From 18665e09bed26f34b57e021252cb4a3601b49151 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 27 Oct 2026 23:31:51 -0700 Subject: [PATCH] refactor(model-cost): load litellm.model_cost through the Rust catalog The Rust catalog parses, validates and installs the cost map into the dict bound to litellm.model_cost. Fetched rows patch the bundled rows, rejected rows fall back to bundled, and fields the user wrote stay theirs. Co-Authored-By: Claude Opus 5.5 --- ci_cd/cost_map_guard.py | 33 ++ litellm-rust/Cargo.lock | 3 + .../crates/model-catalog/src/catalog.rs | 103 +++++ .../crates/model-catalog/src/layers.rs | 139 +++++- .../crates/model-catalog/tests/catalog.rs | 1 + litellm-rust/crates/python-bridge/Cargo.toml | 3 + litellm-rust/crates/python-bridge/src/lib.rs | 100 +++- .../python-bridge/src/model_registry.rs | 298 ++++++++++++ .../src/model_registry/install.rs | 234 ++++++++++ .../src/model_registry/row_error.rs | 117 +++++ litellm/__init__.py | 9 +- litellm/_catalog/__init__.py | 10 + litellm/cost_calculator.py | 2 +- .../litellm_core_utils/get_model_cost_map.py | 387 ++++------------ .../llm_cost_calc/tool_call_cost_tracking.py | 3 +- .../image_generation/cost_calculator.py | 3 +- litellm/llms/bedrock/common_utils.py | 24 +- .../llms/openai/responses/transformation.py | 5 +- litellm/llms/vertex_ai/common_utils.py | 24 +- litellm/llms/vertex_ai/vertex_llm_base.py | 12 +- litellm/proxy/auth/auth_checks.py | 4 +- litellm/proxy/proxy_server.py | 9 +- litellm/router.py | 6 +- litellm/rust_bridge/_native.pyi | 69 ++- litellm/utils.py | 32 +- pyproject.toml | 2 + .../test_get_llm_provider_logic.py | 19 +- .../test_get_model_cost_map.py | 433 ++++++++++-------- .../unit/litellm_core_utils/test_tokenizer.py | 10 +- .../test_azure_ai_flux2_image_generation.py | 4 +- .../test_agentcore_search_transformation.py | 6 +- .../llms/bedrock/test_bedrock_common_utils.py | 21 +- .../test_moonshot_chat_transformation.py | 9 +- .../common_utils/test_config_sync_pubsub.py | 11 +- .../test_routes_model_cost_map.py | 27 +- ...test_proxy_server_endpoints_and_startup.py | 52 ++- tests/unit/search/test_main.py | 3 +- tests/unit/test_claude_fable_5_config.py | 4 +- tests/unit/test_claude_haiku_5_5_config.py | 4 +- tests/unit/test_claude_opus_5_config.py | 4 +- tests/unit/test_claude_sonnet_5_config.py | 4 +- tests/unit/test_cost_map_guard.py | 24 +- tests/unit/test_model_cost_aliases.py | 63 +-- tests/unit/test_private_usage_aliases.py | 7 - .../test_register_model_custom_pricing.py | 78 +--- tests/unit/test_router/test_router.py | 4 +- 46 files changed, 1665 insertions(+), 754 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/model_registry.rs create mode 100644 litellm-rust/crates/python-bridge/src/model_registry/install.rs create mode 100644 litellm-rust/crates/python-bridge/src/model_registry/row_error.rs create mode 100644 litellm/_catalog/__init__.py diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index c1aeaabc5d2..1d95da149ad 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -5,6 +5,9 @@ checks: the files parse and the backup copy matches the root file. A pull reques skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those two files and may only add or update models. + +Installed wheels layer a fetched map over their bundled one, so a key or a price or ``supports_*`` field the map +leaves out is still served from the wheel. Every pull request that drops one must write it as ``null`` instead. """ from __future__ import annotations @@ -91,6 +94,35 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]: return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)} +def _retired_by_null(field: str) -> bool: + return "cost" in field or field in ("tiered_pricing", "off_peak_pricing") or field.startswith("supports_") + + +def _silent_removal_failures(base: Snapshot, head_map: CostMap) -> tuple[str, ...]: + base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) + if isinstance(base_map, str): + return () + head_entries: Final = _entries(head_map) + removed_fields: Final = tuple( + f"{key}.{field}" + for key, entry in _entries(base_map).items() + if key in head_entries + for field in entry + if _retired_by_null(field) and field not in head_entries[key] + ) + return ( + *( + f"{key} is removed without a null; write it as null so older wheels stop serving it" + for key in base_map + if key not in head_map and key not in SPECIAL_ROOT_KEYS + ), + *( + f"{ref} is removed without a null; write it as null so older wheels stop serving it" + for ref in removed_fields + ), + ) + + def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]: base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) if isinstance(base_map, str): @@ -147,6 +179,7 @@ def guard_failures( *_file_failures( head, head_map, evaluation_output_price_sources=evaluation_output_price_sources ), + *_silent_removal_failures(base, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()), ) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b182bb65507..076785c2dd4 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4268,6 +4268,7 @@ dependencies = [ "criterion", "fancy-regex 0.19.2", "futures-util", + "indexmap 2.14.0", "litellm-auth", "litellm-auth-aws", "litellm-cache", @@ -4287,6 +4288,7 @@ dependencies = [ "litellm-inference-transcription", "litellm-llms", "litellm-llms-types", + "litellm-model-catalog", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", @@ -4304,6 +4306,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "sha1 0.10.7", "sqlx", "strum", "thiserror 2.0.19", diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index de177de1647..f461e19a329 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -62,6 +62,7 @@ pub struct Catalog { aliases: IndexMap, lowercase_keys: HashMap, fallback_generalizations: Option, + removals: Vec, } impl IntegrityLimits { @@ -96,7 +97,17 @@ impl Catalog { let mut sections = Vec::new(); let mut entries = IndexMap::with_capacity(document.len()); let mut alias_lists = Vec::new(); + let mut removals = Vec::new(); for (index, (name, value)) in document.into_iter().enumerate() { + if value.is_null() && !is_section(&name) { + removals.push(name.clone()); + sections.push(Section { + index, + key: name, + value, + }); + continue; + } let fields = match value { Value::Object(fields) if !is_section(&name) => fields, value => { @@ -169,6 +180,7 @@ impl Catalog { aliases, lowercase_keys, fallback_generalizations, + removals, }, errors: problems, }) @@ -280,6 +292,45 @@ impl Catalog { (catalog, removed_errors) } + /// Rows replaced by the same key's row in `bundled` when it has one, and left out otherwise (decision 15) + pub fn with_rejected_replaced<'a>( + self, + errors: &'a [Located], + bundled: &Catalog, + ) -> (Self, Vec>) { + let (mut catalog, removed) = self.without_rejected(errors); + let mut rejected: IndexMap> = IndexMap::new(); + for error in removed { + if let Some(key) = crate::error::pointer_root(&error.path) { + rejected.entry(key).or_default().push(error); + } + } + let rejections = rejected + .into_iter() + .map(|(key, errors)| { + let bundled_entry = bundled.entries.get(&key).cloned(); + let replaced_by_bundled = bundled_entry.is_some(); + if let Some(entry) = bundled_entry { + catalog + .lowercase_keys + .insert(key.to_lowercase(), key.clone()); + catalog.entries.insert(key.clone(), entry); + } + Rejection { + key, + errors, + replaced_by_bundled, + } + }) + .collect(); + (catalog, rejections) + } + + /// Keys written as `null`, which a fetched document uses to retire a row it no longer serves (decision 23) + pub fn removals(&self) -> impl Iterator { + self.removals.iter().map(String::as_str) + } + pub fn unrecognized(&self) -> impl Iterator { self.entries.iter().flat_map(|(name, entry)| { entry @@ -370,3 +421,55 @@ fn unknown_timezone(wire: &EntryWire) -> Option { ) }) } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn parse(document: Value) -> Loaded { + Catalog::parse(document.to_string().as_bytes()).unwrap() + } + + #[rstest] + fn a_rejected_row_is_replaced_by_bundled_or_left_out() { + let bundled = + parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}})) + .value; + let fetched = parse(json!({ + "gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1}, + "acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"}, + "kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0} + })); + + let (catalog, rejections) = fetched + .value + .with_rejected_replaced(&fetched.errors, &bundled); + + let replaced: Vec<(&str, bool)> = rejections + .iter() + .map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled)) + .collect(); + assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]); + assert_eq!( + catalog + .lookup("gpt-4o") + .unwrap() + .entry + .field("input_cost_per_token"), + Some(&json!(1.0)) + ); + assert!(catalog.lookup("acme/new").is_none()); + assert!(catalog.lookup("kept").is_some()); + } + + #[rstest] + fn a_null_row_is_a_removal_not_an_error() { + let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null})); + assert!(loaded.errors.is_empty(), "{:?}", loaded.errors); + assert_eq!(loaded.value.removals().collect::>(), ["retired"]); + assert_eq!(loaded.value.served().count(), 1); + } +} diff --git a/litellm-rust/crates/model-catalog/src/layers.rs b/litellm-rust/crates/model-catalog/src/layers.rs index 2b9e3121f0c..5440911c230 100644 --- a/litellm-rust/crates/model-catalog/src/layers.rs +++ b/litellm-rust/crates/model-catalog/src/layers.rs @@ -7,6 +7,8 @@ use crate::catalog::Catalog; use crate::register::Registrations; use crate::wire::Fields; +const ALIASES: &str = "aliases"; + /// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove` #[derive(Clone, Debug, PartialEq)] pub enum Patch { @@ -16,10 +18,6 @@ pub enum Patch { /// The four layers served rows are composed from, lowest first (decisions 23 and 25) #[derive(Clone, Debug)] -#[expect( - dead_code, - reason = "02 and 07 add the functions that read these layers" -)] pub struct Layers { /// 0, the wheel's embedded map bundled: Arc, @@ -28,5 +26,138 @@ pub struct Layers { /// 2, `register(persist=False)`, cleared by every install and refilled by the router replay transient: IndexMap, /// 3, `register(persist=True)`, applied over every install + #[expect(dead_code, reason = "07 applies registrations over the fetched layer")] persisted: Registrations, } + +impl Patch { + pub fn from_value(value: &Value) -> Self { + match value { + Value::Null => Self::Remove, + value => Self::Set(value.clone()), + } + } +} + +impl Layers { + pub fn new(bundled: Arc) -> Self { + Self { + bundled, + fetched: None, + transient: IndexMap::new(), + persisted: Registrations::default(), + } + } + + /// Install replaces layer 1 and clears layer 2, which the router replay refills + pub fn replace_fetched(&mut self, fetched: Arc) { + self.fetched = Some(fetched); + self.transient.clear(); + } + + /// What install writes. A fetched row patches the bundled row field by field, a fetched `null` row removes it + pub fn compose(&self) -> IndexMap { + let mut rows: IndexMap = self + .bundled + .served() + .map(|(key, fields)| (key.to_owned(), without_aliases(fields))) + .collect(); + let Some(fetched) = &self.fetched else { + return rows; + }; + for key in fetched.removals() { + rows.shift_remove(key); + } + for (key, fields) in fetched.served() { + let row = rows.entry(key.to_owned()).or_default(); + for (field, value) in fields.iter().filter(|(field, _)| *field != ALIASES) { + match Patch::from_value(value) { + Patch::Set(value) => { + row.insert(field.clone(), value); + } + Patch::Remove => { + row.shift_remove(field); + } + } + } + } + rows + } + + /// The canonical key an alias is served under, from the fetched layer first + pub fn alias_owner(&self, key: &str) -> Option<&str> { + self.fetched + .iter() + .chain([&self.bundled]) + .find_map(|catalog| catalog.aliases().get(key)) + .map(String::as_str) + } +} + +fn without_aliases(fields: &Fields) -> Fields { + fields + .iter() + .filter(|(field, value)| *field != ALIASES && !value.is_null()) + .map(|(field, value)| (field.clone(), value.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn catalog(document: Value) -> Arc { + Arc::new( + Catalog::parse(document.to_string().as_bytes()) + .unwrap() + .value, + ) + } + + fn bundled() -> Arc { + catalog(json!({ + "gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384}, + "retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0} + })) + } + + fn composed(fetched: Value) -> IndexMap { + let mut layers = Layers::new(bundled()); + layers.replace_fetched(catalog(fetched)); + layers.compose() + } + + #[rstest] + fn a_field_the_fetched_row_lacks_is_served_from_bundled() { + let rows = composed( + json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}), + ); + assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0)); + assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384)); + } + + #[rstest] + fn a_null_field_removes_it() { + let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}})); + assert!(!rows["gpt-4o"].contains_key("max_tokens")); + } + + #[rstest] + fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() { + let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}})); + let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null})); + assert!(kept.contains_key("retired")); + assert!(!retired.contains_key("retired")); + } + + #[rstest] + fn aliases_are_served_under_every_name_without_the_aliases_field() { + let rows = + composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}})); + assert_eq!(rows["new-alias"], rows["new"]); + assert!(!rows["new"].contains_key("aliases")); + } +} diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index e432408dce2..9f341075d95 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -380,6 +380,7 @@ fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() { assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors); let unrecognized: Vec<_> = catalog.unrecognized().collect(); assert!(unrecognized.is_empty(), "{unrecognized:#?}"); + assert!(catalog.removals().next().is_none()); assert!( catalog .fallback_rules() diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 8afde8362f6..40301a71431 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -48,6 +48,9 @@ litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-host-python.workspace = true +litellm-model-catalog.workspace = true +indexmap = "2.14.0" +sha1.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } pyo3.workspace = true prost.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12827b76ff3..462ded582c6 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -8,6 +8,7 @@ mod execution; mod http; mod logger; mod marshal; +mod model_registry; mod preflight; mod python_settings; mod routes; @@ -18,7 +19,10 @@ mod tokenizer; mod _native { #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; - use pyo3::{prelude::*, types::PyModule}; + use pyo3::{ + prelude::*, + types::{PyModule, PyTuple}, + }; #[cfg(feature = "panic-test")] #[pymodule_export] @@ -30,6 +34,8 @@ mod _native { #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] + use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable}; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -66,7 +72,18 @@ mod _native { dict.set_item( "_SecretManagerRuntime", py.get_type::(), - ) + )?; + let typing = py.import("typing")?; + let literal = typing.getattr("Literal")?; + let row_error_kinds = PyTuple::new( + py, + ::VARIANTS + .iter() + .map(|kind| literal.get_item(<&'static str>::from(*kind))) + .collect::>>()?, + )?; + let row_error_kind = typing.getattr("Union")?.get_item(row_error_kinds)?; + module.add("RowErrorKind", row_error_kind) } } @@ -81,6 +98,35 @@ pub(crate) fn native_module(py: Python<'_>) -> Bound<'_, PyModule> { mod tests { use super::*; + fn singleton_literal_values(py: Python<'_>, alias: &Bound<'_, PyAny>) -> (bool, Vec) { + let typing = py.import("typing").unwrap(); + let literal = typing.getattr("Literal").unwrap(); + let components = typing.call_method1("get_args", (alias,)).unwrap(); + let inspected = components + .try_iter() + .unwrap() + .map(|component| { + let component = component.unwrap(); + let origin = typing.call_method1("get_origin", (&component,)).unwrap(); + let arguments = typing.call_method1("get_args", (&component,)).unwrap(); + let values = arguments.extract::>().unwrap(); + ( + origin.is(&literal) && values.len() == 1, + values.first().cloned(), + ) + }) + .collect::>(); + ( + inspected + .iter() + .all(|(is_singleton_literal, _)| *is_singleton_literal), + inspected + .into_iter() + .filter_map(|(_, value)| value) + .collect(), + ) + } + #[rstest::rstest] fn module_registration_preserves_the_public_surface() { Python::initialize(); @@ -107,6 +153,12 @@ mod tests { "aresponses", "ResponsesWebSocketConnection", "NativeDiagnosticProcessor", + "Catalog", + "Fetched", + "Installed", + "Unavailable", + "RowError", + "RowErrorKind", "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "trace_encode_error", @@ -132,4 +184,48 @@ mod tests { assert_eq!(public_names, expected); }); } + + #[rstest::rstest] + fn row_error_kind_is_a_union_of_singleton_literals() { + Python::initialize(); + Python::attach(|py| { + let module = native_module(py); + let typing = py.import("typing").unwrap(); + let union = typing.getattr("Union").unwrap(); + let alias = module.getattr("RowErrorKind").unwrap(); + let origin = typing.call_method1("get_origin", (alias.clone(),)).unwrap(); + assert!(origin.is(&union)); + let (all_singleton_literals, actual) = singleton_literal_values(py, &alias); + assert!(all_singleton_literals); + assert_eq!( + actual, + Vec::from([ + "json".to_owned(), + "empty".to_owned(), + "below_minimum".to_owned(), + "shrunk".to_owned(), + "not_an_object".to_owned(), + "invalid".to_owned(), + "unknown_block_key".to_owned(), + "missing_provider".to_owned(), + "unknown_timezone".to_owned(), + "alias_shadows_model".to_owned(), + "alias_taken".to_owned(), + "duplicate_price".to_owned(), + "suffix_inside_block".to_owned(), + "tier_range".to_owned(), + "tier_reversed".to_owned(), + "tier_gap".to_owned(), + "mixed_tier_basis".to_owned(), + "not_priced_by_results".to_owned(), + "bands_from_two_sources".to_owned(), + "not_an_off_peak_rate".to_owned(), + "no_off_peak_prices".to_owned(), + "no_off_peak_hours".to_owned(), + "empty_list".to_owned(), + "invalid_date".to_owned(), + ]) + ); + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs new file mode 100644 index 00000000000..6dead98763e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -0,0 +1,298 @@ +mod install; +mod row_error; + +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, Mutex, PoisonError}; + +use litellm_host_python::{release_gil, to_py}; +use litellm_model_catalog::wire::Fields; +use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers}; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::*; +use pyo3::pybacked::PyBackedBytes; +use pyo3::types::{PyDict, PyModule}; +use serde::Serialize; +use sha1::{Digest, Sha1}; + +use install::{Record, write_into}; +pub(crate) use row_error::{RowError, RowErrorKind, row_error}; + +static BUNDLED: LazyLock> = LazyLock::new(|| { + let loaded = Catalog::parse(BUNDLED_REGISTRY).expect("the bundled model registry parses"); + let catalog = Arc::new(loaded.value); + Arc::new(Parsed { + raw: Arc::clone(&catalog), + catalog, + errors: loaded.errors.iter().map(row_error).collect(), + rejected: Vec::new(), + }) +}); + +static BUNDLED_REVISION: LazyLock = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY)); + +struct Parsed { + /// What install layers over bundled, rows decision 15 rejects replaced or left out + catalog: Arc, + /// The document as written + raw: Arc, + errors: Vec, + rejected: Vec, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Fetched { + parsed: Arc, + #[pyo3(get)] + revision: String, + #[pyo3(get)] + etag: Option, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Installed { + /// Rows the composed map serves, what main's adopt returns as the fetched map's length + #[pyo3(get)] + count: usize, + /// Keys set because the row differed or was new + #[pyo3(get)] + written: usize, + /// Keys deleted because the map no longer serves them + #[pyo3(get)] + removed: usize, + /// Decision 15, empty for the bundled map + #[pyo3(get)] + rejected: Vec, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Unavailable { + #[pyo3(get)] + reason: String, +} + +struct State { + layers: Layers, + record: Option, +} + +#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")] +pub(crate) struct CatalogHandle { + module: Py, + attr: String, + state: Mutex, +} + +#[pymethods] +impl Fetched { + /// The rows of this one document as install would serve them, without layers + fn served(&self, py: Python<'_>) -> PyResult> { + let served = release_gil(py, || served_rows(&self.parsed.catalog)); + let rows = PyDict::new(py); + let mut shared: HashMap> = HashMap::new(); + for (key, owner, fields) in served { + let row = match shared.get(&owner) { + Some(row) => row.clone_ref(py), + None => { + let row = fields_to_dict(py, &fields)?; + shared.insert(owner, row.clone_ref(py)); + row + } + }; + rows.set_item(key, row)?; + } + Ok(rows.unbind()) + } + + /// The raw document, `sample_spec` and `fallback_generalizations` included + fn document(&self, py: Python<'_>) -> PyResult> { + to_py(py, &WrittenDocument(self.parsed.raw.as_ref())) + } + + fn fallback_rules(&self, py: Python<'_>) -> PyResult> { + to_py(py, &self.parsed.raw.fallback_rules()) + } + + fn errors(&self) -> Vec { + self.parsed.errors.clone() + } +} + +#[pymethods] +impl CatalogHandle { + #[staticmethod] + fn bound_to(module: Bound<'_, PyModule>, attr: String) -> Self { + Self { + module: module.unbind(), + attr, + state: Mutex::new(State { + layers: Layers::new(Arc::clone(&BUNDLED.catalog)), + record: None, + }), + } + } + + fn bundled(&self) -> Fetched { + Fetched { + parsed: Arc::clone(&BUNDLED), + revision: BUNDLED_REVISION.clone(), + etag: None, + } + } + + /// Parses a fetched body and applies main's whole-map checks against the bundled model count + #[staticmethod] + #[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))] + fn parse( + py: Python<'_>, + body: PyBackedBytes, + etag: Option, + min_model_count: usize, + max_shrink_ratio: f64, + ) -> PyResult> { + let parsed = release_gil(py, || { + parse_remote(&body, min_model_count, max_shrink_ratio) + }); + match parsed { + Ok((parsed, revision)) => Ok(Fetched { + parsed: Arc::new(parsed), + revision, + etag, + } + .into_pyobject(py)? + .into_any() + .unbind()), + Err(reason) => Ok(Unavailable { reason } + .into_pyobject(py)? + .into_any() + .unbind()), + } + } + + /// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25) + fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult { + let bound = self.module.bind(py).getattr(self.attr.as_str())?; + let dict = bound + .cast::() + .map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?; + let dict_id = dict.as_ptr() as usize; + let (layers, record) = { + let mut state = self.lock(); + state + .layers + .replace_fetched(Arc::clone(&fetched.parsed.catalog)); + let record = state + .record + .clone() + .filter(|record| record.dict_id == dict_id); + (state.layers.clone(), record) + }; + let composed = release_gil(py, || layers.compose()); + let outcome = write_into(py, dict, &composed, &layers, record)?; + self.lock().record = Some(outcome.record); + Ok(Installed { + count: composed.len(), + written: outcome.written, + removed: outcome.removed, + rejected: fetched.parsed.rejected.clone(), + }) + } +} + +impl CatalogHandle { + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +fn parse_remote( + body: &[u8], + min_model_count: usize, + max_shrink_ratio: f64, +) -> Result<(Parsed, String), String> { + let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio) + .ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?; + let loaded = Catalog::parse(body).map_err(|error| error.to_string())?; + loaded + .value + .check_size(BUNDLED.raw.model_count(), limits) + .map_err(|error| error.to_string())?; + let raw = Arc::new(loaded.value.clone()); + let errors = loaded.errors.iter().map(row_error).collect(); + let (catalog, rejections) = loaded + .value + .with_rejected_replaced(&loaded.errors, &BUNDLED.raw); + let rejected = rejections + .iter() + .flat_map(|rejection| rejection.errors.iter().copied().map(row_error)) + .collect(); + Ok(( + Parsed { + catalog: Arc::new(catalog), + raw, + errors, + rejected, + }, + git_blob_id(body), + )) +} + +fn git_blob_id(body: &[u8]) -> String { + let mut hasher = Sha1::new(); + hasher.update(format!("blob {}\0", body.len()).as_bytes()); + hasher.update(body); + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> { + catalog + .served() + .map(|(key, fields)| { + let owner = catalog + .aliases() + .get(key) + .cloned() + .unwrap_or_else(|| key.to_owned()); + let fields = fields + .iter() + .filter(|(field, _)| *field != "aliases") + .map(|(field, value)| (field.clone(), value.clone())) + .collect(); + (key.to_owned(), owner, fields) + }) + .collect() +} + +fn fields_to_dict(py: Python<'_>, fields: &Fields) -> PyResult> { + let row = PyDict::new(py); + for (key, value) in fields { + row.set_item(key, to_py(py, value)?)?; + } + Ok(row.unbind()) +} + +struct WrittenDocument<'a>(&'a Catalog); + +impl Serialize for WrittenDocument<'_> { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_map(self.0.written()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn the_revision_is_the_git_blob_id_of_the_body() { + assert_eq!( + git_blob_id(b"hello\n"), + "ce013625030ba8dba906f756967f9e9ca394464a" + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/model_registry/install.rs b/litellm-rust/crates/python-bridge/src/model_registry/install.rs new file mode 100644 index 00000000000..b019dab922b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry/install.rs @@ -0,0 +1,234 @@ +use std::collections::HashMap; + +use indexmap::IndexMap; +use litellm_host_python::{from_py, to_py}; +use litellm_model_catalog::Layers; +use litellm_model_catalog::wire::Fields; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use serde_json::Value; + +/// The field values litellm last wrote into one dict object (decision 25) +#[derive(Clone, Debug)] +pub(crate) struct Record { + pub(crate) dict_id: usize, + rows: HashMap, +} + +pub(crate) struct Outcome { + pub(crate) record: Record, + pub(crate) written: usize, + pub(crate) removed: usize, +} + +/// What the dict holds for one field, as litellm compares it +#[derive(PartialEq)] +enum Held { + Absent, + Json(Value), + /// A value that does not convert to JSON, which only the user can have written + Foreign, +} + +/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the +/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the +/// last install is written in full, which is main's result after rebind then adopt +pub(crate) fn write_into( + py: Python<'_>, + dict: &Bound<'_, PyDict>, + composed: &IndexMap, + layers: &Layers, + previous: Option, +) -> PyResult { + let dict_id = dict.as_ptr() as usize; + let Some(previous) = previous else { + return write_in_full(py, dict, composed, layers, dict_id); + }; + let mut rows = previous.rows; + let mut written = 0; + let mut removed = 0; + let mut fresh: HashMap> = HashMap::new(); + let keys: Vec = composed + .keys() + .chain(rows.keys().filter(|key| !composed.contains_key(*key))) + .cloned() + .collect(); + for key in keys { + let target = composed.get(&key); + let last = rows.remove(&key); + let Some(current) = dict.get_item(&key)? else { + match (target, last) { + (Some(target), None) => { + dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?; + rows.insert(key, target.clone()); + written += 1; + } + (_, Some(last)) if target.is_some() => { + rows.insert(key, last); + } + _ => {} + } + continue; + }; + let Ok(row) = current.cast::() else { + if let Some(last) = last { + rows.insert(key, last); + } + continue; + }; + let last = last.unwrap_or_default(); + let (owned, changed) = write_owned_fields(py, row, target, &last)?; + if target.is_none() && changed && row.is_empty() { + dict.del_item(&key)?; + removed += 1; + continue; + } + written += usize::from(changed); + if !owned.is_empty() { + rows.insert(key, owned); + } + } + Ok(Outcome { + record: Record { dict_id, rows }, + written, + removed, + }) +} + +fn write_in_full( + py: Python<'_>, + dict: &Bound<'_, PyDict>, + composed: &IndexMap, + layers: &Layers, + dict_id: usize, +) -> PyResult { + let stale: Vec> = dict + .keys() + .iter() + .filter(|key| { + key.extract::() + .map_or(true, |key| !composed.contains_key(&key)) + }) + .collect(); + let removed = stale.len(); + for key in stale { + dict.del_item(key)?; + } + let mut fresh: HashMap> = HashMap::new(); + let mut written = 0; + for (key, target) in composed { + let current = dict.get_item(key)?; + let row = current.as_ref().and_then(|row| row.cast::().ok()); + match row { + Some(row) => { + let owned_by_litellm: Fields = row + .iter() + .filter_map(|(field, value)| { + let field = field.extract::().ok()?; + Some((field, from_py::(&value).ok()?)) + }) + .collect(); + let extra: Vec = row + .keys() + .iter() + .filter_map(|field| field.extract::().ok()) + .filter(|field| !owned_by_litellm.contains_key(field)) + .collect(); + for field in extra { + row.del_item(field)?; + } + let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?; + written += usize::from(changed); + } + None => { + dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?; + written += 1; + } + } + } + Ok(Outcome { + record: Record { + dict_id, + rows: composed + .iter() + .map(|(key, fields)| (key.clone(), fields.clone())) + .collect(), + }, + written, + removed, + }) +} + +/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed +fn write_owned_fields( + py: Python<'_>, + row: &Bound<'_, PyDict>, + target: Option<&Fields>, + last: &Fields, +) -> PyResult<(Fields, bool)> { + let empty = Fields::new(); + let target = target.unwrap_or(&empty); + let fields: Vec<&String> = target + .keys() + .chain(last.keys().filter(|field| !target.contains_key(*field))) + .collect(); + let mut owned = Fields::new(); + let mut changed = false; + for field in fields { + let held = held(row, field)?; + let litellm_owns = match (&held, last.get(field)) { + (Held::Absent, None) => true, + (Held::Json(value), Some(last)) => value == last, + _ => false, + }; + if !litellm_owns { + if let Some(last) = last.get(field) { + owned.insert(field.clone(), last.clone()); + } + continue; + } + match target.get(field) { + Some(value) => { + if held != Held::Json(value.clone()) { + row.set_item(field, to_py(py, value)?)?; + changed = true; + } + owned.insert(field.clone(), value.clone()); + } + None if held != Held::Absent => { + row.del_item(field)?; + changed = true; + } + None => {} + } + } + Ok((owned, changed)) +} + +fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult { + Ok(match row.get_item(field)? { + None => Held::Absent, + Some(value) => from_py::(&value).map_or(Held::Foreign, Held::Json), + }) +} + +/// A new row, the same object for every key of one alias group, as main's alias expansion gives them +fn shared_row( + py: Python<'_>, + fresh: &mut HashMap>, + layers: &Layers, + key: &str, + fields: &Fields, +) -> PyResult> { + let owner = layers.alias_owner(key).unwrap_or(key).to_owned(); + if let Some(row) = fresh.get(&owner) { + return Ok(row.clone_ref(py)); + } + let row = PyDict::new(py); + for (field, value) in fields { + row.set_item(field, to_py(py, value)?)?; + } + let row = row.unbind(); + fresh.insert(owner, row.clone_ref(py)); + Ok(row) +} diff --git a/litellm-rust/crates/python-bridge/src/model_registry/row_error.rs b/litellm-rust/crates/python-bridge/src/model_registry/row_error.rs new file mode 100644 index 00000000000..99bd7f1595f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry/row_error.rs @@ -0,0 +1,117 @@ +use litellm_model_catalog::{Error, Located}; +use pyo3::prelude::*; +use strum::{IntoStaticStr, VariantArray}; + +#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)] +pub(crate) enum RowErrorKind { + #[strum(serialize = "json")] + Json, + #[strum(serialize = "empty")] + Empty, + #[strum(serialize = "below_minimum")] + BelowMinimum, + #[strum(serialize = "shrunk")] + Shrunk, + #[strum(serialize = "not_an_object")] + NotAnObject, + #[strum(serialize = "invalid")] + Invalid, + #[strum(serialize = "unknown_block_key")] + UnknownBlockKey, + #[strum(serialize = "missing_provider")] + MissingProvider, + #[strum(serialize = "unknown_timezone")] + UnknownTimezone, + #[strum(serialize = "alias_shadows_model")] + AliasShadowsModel, + #[strum(serialize = "alias_taken")] + AliasTaken, + #[strum(serialize = "duplicate_price")] + DuplicatePrice, + #[strum(serialize = "suffix_inside_block")] + SuffixInsideBlock, + #[strum(serialize = "tier_range")] + TierRange, + #[strum(serialize = "tier_reversed")] + TierReversed, + #[strum(serialize = "tier_gap")] + TierGap, + #[strum(serialize = "mixed_tier_basis")] + MixedTierBasis, + #[strum(serialize = "not_priced_by_results")] + NotPricedByResults, + #[strum(serialize = "bands_from_two_sources")] + BandsFromTwoSources, + #[strum(serialize = "not_an_off_peak_rate")] + NotAnOffPeakRate, + #[strum(serialize = "no_off_peak_prices")] + NoOffPeakPrices, + #[strum(serialize = "no_off_peak_hours")] + NoOffPeakHours, + #[strum(serialize = "empty_list")] + EmptyList, + #[strum(serialize = "invalid_date")] + InvalidDate, +} + +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone, Debug)] +pub(crate) struct RowError { + #[pyo3(get)] + pub(crate) key: String, + #[pyo3(get)] + pub(crate) field: Option, + #[pyo3(get)] + pub(crate) kind: String, + #[pyo3(get)] + pub(crate) rejects: bool, + #[pyo3(get)] + pub(crate) message: String, +} + +pub(crate) fn row_error(error: &Located) -> RowError { + let mut path = error.path.split('/').skip(1); + let key = unescape_pointer_segment(path.next().unwrap_or_default()); + let field = path.map(unescape_pointer_segment).collect::>(); + let field = (!field.is_empty()).then(|| field.join("/")); + RowError { + key, + field, + kind: <&'static str>::from(error_kind(&error.error)).to_owned(), + rejects: litellm_model_catalog::rejects_row(&error.error), + message: error.error.to_string(), + } +} + +fn unescape_pointer_segment(segment: &str) -> String { + segment.replace("~1", "/").replace("~0", "~") +} + +fn error_kind(error: &Error) -> RowErrorKind { + match error { + Error::Json(_) => RowErrorKind::Json, + Error::Empty => RowErrorKind::Empty, + Error::BelowMinimum { .. } => RowErrorKind::BelowMinimum, + Error::Shrunk { .. } => RowErrorKind::Shrunk, + Error::NotAnObject => RowErrorKind::NotAnObject, + Error::Invalid(_) => RowErrorKind::Invalid, + Error::UnknownBlockKey => RowErrorKind::UnknownBlockKey, + Error::MissingProvider => RowErrorKind::MissingProvider, + Error::UnknownTimezone { .. } => RowErrorKind::UnknownTimezone, + Error::AliasShadowsModel { .. } => RowErrorKind::AliasShadowsModel, + Error::AliasTaken { .. } => RowErrorKind::AliasTaken, + Error::DuplicatePrice { .. } => RowErrorKind::DuplicatePrice, + Error::SuffixInsideBlock { .. } => RowErrorKind::SuffixInsideBlock, + Error::TierRange => RowErrorKind::TierRange, + Error::TierReversed => RowErrorKind::TierReversed, + Error::TierGap { .. } => RowErrorKind::TierGap, + Error::MixedTierBasis => RowErrorKind::MixedTierBasis, + Error::NotPricedByResults => RowErrorKind::NotPricedByResults, + Error::BandsFromTwoSources => RowErrorKind::BandsFromTwoSources, + Error::NotAnOffPeakRate => RowErrorKind::NotAnOffPeakRate, + Error::NoOffPeakPrices => RowErrorKind::NoOffPeakPrices, + Error::NoOffPeakHours => RowErrorKind::NoOffPeakHours, + Error::EmptyList => RowErrorKind::EmptyList, + Error::InvalidDate => RowErrorKind::InvalidDate, + } +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 41e3fa34323..bbdb2ac0d67 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -548,9 +548,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None #### PII MASKING #### output_parse_pii: bool = False ############################################# -from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete +from litellm.litellm_core_utils.get_model_cost_map import ( + get_model_cost_map as get_model_cost_map, + install_initial_model_cost_map, + mark_litellm_import_complete, +) -model_cost = get_model_cost_map(url=model_cost_map_url) +model_cost: dict[str, dict[str, object]] = {} +install_initial_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount apply_search_free_tier: bool = False cost_margin_config: Dict[ diff --git a/litellm/_catalog/__init__.py b/litellm/_catalog/__init__.py new file mode 100644 index 00000000000..0b9c4f653cb --- /dev/null +++ b/litellm/_catalog/__init__.py @@ -0,0 +1,10 @@ +"""Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release""" + +import sys +from typing import Final + +from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable + +CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost") + +__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 86b22069a39..bc1bdf64825 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2794,7 +2794,7 @@ def default_image_cost_calculator( price * units for price_table in price_tables for cost_key, units in unit_counts - if (price := price_table.get(cost_key)) is not None + if isinstance(price := price_table.get(cost_key), (int, float)) ), None, ) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 7fc6eab6c60..e727bb8eff1 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -10,8 +10,6 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True """ import asyncio -import hashlib -import json import os import random import sys @@ -19,19 +17,19 @@ import threading import time import urllib.parse from collections.abc import Awaitable, Callable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass from datetime import datetime, timezone -from importlib.resources import files from itertools import chain from pathlib import Path from types import MappingProxyType from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import JsonValue, TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm import verbose_logger +from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable from litellm.constants import ( MODEL_COST_MAP_MAX_SHRINK_RATIO, MODEL_COST_MAP_MIN_MODEL_COUNT, @@ -43,8 +41,6 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) -_DECODED_JSON: Final = TypeAdapter(object) -_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -52,49 +48,8 @@ def _is_cli_process() -> bool: return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES -# Reserved top-level keys that are not model entries. They must be excluded -# from the model-count integrity check so a real upstream shrink can't be masked. -RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY}) - - -def _count_model_entries(model_cost: dict) -> int: - """Count actual model entries, excluding reserved meta keys.""" - return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS) - - -def git_blob_id(body: bytes) -> str: - return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() - - class GetModelCostMap: - """ - Handles fetching, validating, and loading the model cost map. - - Only the backup model *count* is cached (a single int). The full - backup dict is never held in memory — it is only parsed when it - needs to be *returned* as a fallback. - """ - - _backup_model_count: int = -1 # -1 = not yet loaded - - @staticmethod - def read_local_model_cost_map_bytes() -> bytes: - return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes() - - @staticmethod - def read_local_model_cost_map_text() -> str: - return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8") - - @staticmethod - def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded": - body: Final = GetModelCostMap.read_local_model_cost_map_bytes() - content: Final = _BUNDLED_CATALOG_ADAPTER.validate_python(json.loads(body)) - return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body)) - - @staticmethod - def load_local_model_cost_map() -> dict: - """Load the local backup model cost map bundled with the package.""" - return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map + """The catalog as last loaded, for readers that want it without later registrations""" _loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({}) _loaded_catalog_fields: frozenset[str] = frozenset() @@ -115,102 +70,6 @@ class GetModelCostMap: cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()}) cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values())) - @classmethod - def get_backup_model_count(cls) -> int: - """Return the number of models in the local backup (cached int).""" - if cls._backup_model_count < 0: - backup: Final = cls.load_local_model_cost_map() - cls._backup_model_count = _count_model_entries(backup) - return cls._backup_model_count - - _get_backup_model_count = get_backup_model_count - - @staticmethod - def _check_is_valid_dict(fetched_map: dict) -> bool: - """Check 1: fetched map is a non-empty dict.""" - if not isinstance(fetched_map, dict): - verbose_logger.warning( - "LiteLLM: Fetched model cost map is not a dict (type=%s). Falling back to local backup.", - type(fetched_map).__name__, - ) - return False - - if len(fetched_map) == 0: - verbose_logger.warning( - "LiteLLM: Fetched model cost map is empty. Falling back to local backup.", - ) - return False - - return True - - @classmethod - def _check_model_count_not_reduced( - cls, - fetched_map: dict, - backup_model_count: int, - min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, - max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, - ) -> bool: - """Check 2: model count has not reduced significantly vs backup.""" - fetched_count: Final = _count_model_entries(fetched_map) - - if fetched_count < min_model_count: - verbose_logger.warning( - "LiteLLM: Fetched model cost map has only %d models (minimum=%d). " - "This may indicate a corrupted upstream file. " - "Falling back to local backup.", - fetched_count, - min_model_count, - ) - return False - - if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio: - verbose_logger.warning( - "LiteLLM: Fetched model cost map shrank significantly " - "(fetched=%d, backup=%d, threshold=%.0f%%). " - "This may indicate a corrupted upstream file. " - "Falling back to local backup.", - fetched_count, - backup_model_count, - max_shrink_ratio * 100, - ) - return False - - return True - - @classmethod - def validate_model_cost_map( - cls, - fetched_map: dict, - backup_model_count: int, - min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, - max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, - ) -> bool: - """ - Validate the integrity of a fetched model cost map. - - Runs each check in order and returns False on the first failure. - - Checks: - 1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict. - 2. ``_check_model_count_not_reduced`` -- model count meets minimum - and has not shrunk >``max_shrink_ratio`` vs backup. - - Returns True if all checks pass, False otherwise. - """ - if not cls._check_is_valid_dict(fetched_map): - return False - - if not cls._check_model_count_not_reduced( - fetched_map=fetched_map, - backup_model_count=backup_model_count, - min_model_count=min_model_count, - max_shrink_ratio=max_shrink_ratio, - ): - return False - - return True - RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 @@ -224,9 +83,15 @@ def mark_litellm_import_complete() -> None: @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: - model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict - revision: str | None = None - etag: str | None = None + fetched: Fetched + + @property + def revision(self) -> str: + return self.fetched.revision + + @property + def etag(self) -> str | None: + return self.fetched.etag @dataclass(frozen=True, slots=True) @@ -314,16 +179,15 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) - def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome: - try: - parsed: Final = _DECODED_JSON.validate_python(json.loads(content)) - except ValueError as e: - return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") - if not isinstance(parsed, dict): - return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") - model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition - dict[str, object], parsed + parsed: Final = CATALOG.parse( + content, + etag=etag, + min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT, + max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO, ) - return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag) + if isinstance(parsed, Unavailable): + return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}") + return ModelCostMapReloaded(fetched=parsed) def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable: @@ -465,7 +329,7 @@ async def refetch_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()) + return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())) result: Final = await _fetch_remote_model_cost_map_with_retry( url=url, @@ -481,17 +345,12 @@ async def refetch_model_cost_map( result.reason, ) return result - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=GetModelCostMap.get_backup_model_count(), - ): - return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation") _cost_map_source_info.loaded_at = datetime.now(timezone.utc) _cost_map_source_info.source = "remote" _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(result) + return _note_loaded(result) class ModelCostMapSourceInfo: @@ -571,103 +430,46 @@ class AliasedCostEntry(dict[str, object]): self.registered_fields: frozenset[str] = registered_fields -def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]: - return MappingProxyType( - { - owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner)) - for owner in dict.fromkeys(alias_owners.values()) - } +def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: + _cost_map_source_info.source_revision = loaded.fetched.revision + _cost_map_source_info.etag = loaded.fetched.etag + set_fallback_generalizations(loaded.fetched.fallback_rules()) + GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module + loaded.fetched.served() ) + return loaded -def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: - """ - Expand ``aliases`` lists in model cost entries into top-level entries. +class _InstallGeneration: + """Bumped by every install, so a registration racing a reload into the same dict can rebuild its entry""" - Each alias gets a reference to the **same** ``AliasedCostEntry`` as the - canonical entry, which carries every name it is served under. The - ``aliases`` key is removed from the entry so downstream code never sees it. - - If an alias collides with an existing canonical entry the alias is - skipped and a warning is logged. - """ - alias_owners: Final[dict[str, str]] = {} - keys_with_aliases: Final[list[str]] = [] - - for model_name, model_info in model_cost.items(): - aliases: object = model_info.get("aliases") - if aliases is None: - continue - keys_with_aliases.append(model_name) - if not isinstance(aliases, list): - verbose_logger.warning( - "LiteLLM model alias field for '%s' is not a list (got %s) — skipping.", - model_name, - type(aliases).__name__, - ) - continue - if not aliases: - continue - for alias in aliases: - if alias in model_cost: - verbose_logger.warning( - "LiteLLM model alias conflict: alias '%s' (from '%s') " - "already exists as a canonical entry — skipping.", - alias, - model_name, - ) - continue - if alias in alias_owners: - verbose_logger.warning( - "LiteLLM model alias conflict: alias '%s' (from '%s') " - "was already claimed by another entry — skipping.", - alias, - model_name, - ) - continue - alias_owners[alias] = model_name - - # Remove the ``aliases`` key from entries so it doesn't pollute model info - for key in keys_with_aliases: - model_cost[key].pop("aliases", None) - - for owner, names in _alias_groups(alias_owners).items(): - model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names))) - return model_cost + value: int = 0 -def _finalize_model_cost_map(model_cost: dict) -> dict: - """Extract fallback generalizations out of the raw map, then expand aliases. - - The ``fallback_generalizations`` block is installed into the generalizations - module and removed from the map so it is never treated as a model entry. - """ - raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None) - rules: Final = raw.get("rules") if isinstance(raw, dict) else None - set_fallback_generalizations(rules) - return _expand_model_aliases(model_cost) +def model_cost_install_generation() -> int: + return _InstallGeneration.value -def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: - _cost_map_source_info.source_revision = loaded.revision - _cost_map_source_info.etag = loaded.etag - finalized: Final = _finalize_model_cost_map(loaded.model_cost_map) - GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module - return replace(loaded, model_cost_map=finalized) +def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed: + installed: Final = catalog.install(fetched) + _InstallGeneration.value += 1 + if installed.rejected: + verbose_logger.warning( + "LiteLLM: model cost map rows failed validation and are served from the bundled map or left out: %s", + "; ".join(f"{error.key}: {error.message}" for error in installed.rejected), + ) + return installed -def adopt_model_cost_map( - new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract -) -> int: +def adopt_model_cost_map(fetched: Fetched) -> int: import litellm from litellm import utils - litellm.model_cost = new_model_cost_map + installed: Final = _install(fetched) utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation - litellm.add_known_models(model_cost_map=new_model_cost_map) - fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0 + litellm.add_known_models(model_cost_map=litellm.model_cost) utils.reapply_runtime_model_cost_registrations() - return fetched_model_count + return installed.count def _retry_remote_fetch_in_background( @@ -700,45 +502,27 @@ def _retry_remote_fetch_in_background( ) return _litellm_import_complete.wait() - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=GetModelCostMap.get_backup_model_count(), - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - return - finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map + _note_loaded(result) _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None _cost_map_source_info.loaded_at = datetime.now(timezone.utc) - adopt_model_cost_map(finalized) + adopt_model_cost_map(result.fetched) except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) -def get_model_cost_map( +def _pick_model_cost_map( url: str, - timeout: int = 5, - max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, - sleep: Callable[[float], None] = time.sleep, - rng: random.Random | None = None, - client: "_SyncGetClient | None" = None, -) -> dict: + timeout: int, + max_attempts: int, + sleep: Callable[[float], None], + rng: random.Random | None, + client: "_SyncGetClient | None", +) -> Fetched: """ - Public entry point — returns the model cost map dict. - - 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set or this is a ``lite`` / - ``litellm-proxy`` CLI process, uses the local backup only. - 2. Otherwise fetches from ``url``, retrying transient HTTP errors - (429/5xx/transport) with Retry-After-aware backoff in a background - thread, validates integrity, and falls back to the local backup on any - failure. - - Only the backup model count is cached (a single int) for validation. - The full backup dict is only parsed when it must be *returned* as a - fallback — it is never held in memory long-term. + The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a + ``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient + failure, and the bundled map on any failure """ _cost_map_source_info.loaded_at = datetime.now(timezone.utc) # Note: can't use get_secret_bool here — this runs during litellm.__init__ @@ -748,7 +532,7 @@ def get_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False @@ -779,22 +563,39 @@ def get_model_cost_map( ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - content: Final = outcome.model_cost_map - - # Validate using cached count (cheap int comparison, no file I/O) - if not GetModelCostMap.validate_model_cost_map( - fetched_map=content, - backup_model_count=GetModelCostMap.get_backup_model_count(), - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(outcome).model_cost_map + return _note_loaded(outcome).fetched + + +def install_initial_model_cost_map(url: str) -> None: + """Fills ``litellm.model_cost`` at import with the map main's rules pick""" + _install( + _pick_model_cost_map( + url=url, + timeout=5, + max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, + sleep=time.sleep, + rng=None, + client=None, + ) + ) + + +def get_model_cost_map( + url: str, + timeout: int = 5, + max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, + sleep: Callable[[float], None] = time.sleep, + rng: random.Random | None = None, + client: "_SyncGetClient | None" = None, +) -> dict[str, dict[str, JsonValue]]: + """ + Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and + validated by the Rust catalog, and the bundled map is returned on any failure + """ + return _pick_model_cost_map( + url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client + ).served() diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 045869e4547..46d0ea1c6bf 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -805,7 +805,8 @@ class StandardBuiltInToolCostTracking: model_key = model_info.get("key") if isinstance(model_info, dict) else getattr(model_info, "key", None) if model_key and model_key in litellm.model_cost: - return litellm.model_cost[model_key].get("code_interpreter_cost_per_session") + session_cost: Final = litellm.model_cost[model_key].get("code_interpreter_cost_per_session") + return float(session_cost) if isinstance(session_cost, (int, float)) else None except Exception: pass diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index beed3881bdf..53fb755be80 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -18,7 +18,8 @@ def _input_cost_per_pixel(resolved: ModelInfo) -> float: shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None if shared_entry is None: return 0.0 - return shared_entry.get("input_cost_per_pixel") or 0.0 + shared_price: Final = shared_entry.get("input_cost_per_pixel") + return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0 def cost_calculator( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 28888f1a23a..113680c2f00 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -200,13 +200,6 @@ def get_cached_model_info(): return _get_model_info -@functools.lru_cache(maxsize=1) -def _get_local_model_cost_map() -> dict: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - return GetModelCostMap.load_local_model_cost_map() - - def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None: """ Remove and return Anthropic's nested ``output_config.format`` field. @@ -1532,12 +1525,6 @@ def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> flag = model_info.get(key) if isinstance(flag, bool): return flag - - model_cost_key = model_info.get("key") - if isinstance(model_cost_key, str): - local_flag = _get_local_model_cost_map().get(model_cost_key, {}).get(key) - if isinstance(local_flag, bool): - return local_flag return None @@ -1579,16 +1566,7 @@ def _get_bedrock_output_config_effort_ceiling( except Exception: return None - ceiling = model_info.get("bedrock_output_config_effort_ceiling") - if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: - return ceiling - - model_cost_key: Final = model_info.get("key") - if not isinstance(model_cost_key, str): - return None - - local_model_info: Final = _get_local_model_cost_map().get(model_cost_key, {}) - ceiling = local_model_info.get("bedrock_output_config_effort_ceiling") + ceiling: Final = model_info.get("bedrock_output_config_effort_ceiling") if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: return ceiling return None diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 07822bb17b3..3e50da6283b 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -12,7 +12,6 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) @@ -209,7 +208,9 @@ def _bundled_openai_reasoning_models() -> frozenset[str]: map it fetches), and a lagging entry must never strip `reasoning` from a model this release knows accepts it. """ - bundled: Final = _BUNDLED_COST_MAP.validate_json(GetModelCostMap.read_local_model_cost_map_text()) + from litellm._catalog import CATALOG + + bundled: Final = _BUNDLED_COST_MAP.validate_python(CATALOG.bundled().document()) return frozenset( name for name, entry in bundled.items() diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 27f2aedb300..d0453b72701 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -3,7 +3,6 @@ from collections.abc import Callable, Iterable, Mapping, Sequence from copy import deepcopy from datetime import datetime, timezone from enum import Enum -from functools import lru_cache from itertools import accumulate, chain, repeat, takewhile from types import MappingProxyType from typing import Any, Final, Literal, TypeVar, cast, get_type_hints @@ -77,19 +76,6 @@ def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyri return None -@lru_cache(maxsize=1) -def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - return MappingProxyType( - { - model_key: lyria_model_info - for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items() - if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None - } - ) - - def _vertex_ai_lyria_model_key(model: str) -> str: return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" @@ -100,18 +86,12 @@ def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: model_key: Final = _vertex_ai_lyria_model_key(model) - runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) - return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key) + return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: model_key: Final = _vertex_ai_lyria_model_key(model) - runtime_cost: Final = _vertex_ai_lyria_generation_cost( - _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) - ) - if runtime_cost is not None: - return runtime_cost - return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key)) + return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))) class VertexAIError(BaseLLMException): diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0cdd92bb071..b460a04c79e 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -12,6 +12,8 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse +from pydantic import TypeAdapter, ValidationError + import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE @@ -57,6 +59,8 @@ else: GoogleCredentialsObject = Any +_SUPPORTED_REGIONS: Final = TypeAdapter(tuple[str, ...]) + class _VertexCredentialsObject(Protocol): """Structural view of the google-auth credentials handle that this class caches and refreshes.""" @@ -114,8 +118,12 @@ class VertexBase: # Try to get supported_regions directly from model_cost # Check both with and without vertex_ai/ prefix model_key: Final = f"vertex_ai/{model}" if not model.startswith("vertex_ai/") else model - model_info: Final = litellm.model_cost.get(model_key, {}) - supported_regions: Final = model_info.get("supported_regions") + try: + supported_regions: Final = _SUPPORTED_REGIONS.validate_python( + litellm.model_cost.get(model_key, {}).get("supported_regions") or () + ) + except ValidationError: + return vertex_region or "us-central1" if supported_regions and len(supported_regions) > 0: # If user didn't specify region, use the first supported region diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..21af6b6d0a9 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -711,7 +711,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) - """ for deployment in deployments: model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") - if model_id is None: + if not isinstance(model_id, str): continue raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry: @@ -729,7 +729,7 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b """ for deployment in deployments: model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") - if model_id is None: + if not isinstance(model_id, str): continue raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..27b79ef51f4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -211,6 +211,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as _Span from prisma import models as prisma_models + from litellm._catalog import Fetched from litellm.integrations.opentelemetry import OpenTelemetry from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager @@ -5385,10 +5386,10 @@ def resolve_classifier_plugin( return resolved -def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: +def _swap_in_model_cost_map(fetched: "Fetched") -> int: from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map - return adopt_model_cost_map(new_model_cost_map) + return adopt_model_cost_map(fetched) def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]: @@ -9104,7 +9105,7 @@ class ProxyConfig: ) return - models_count = _swap_in_model_cost_map(reload_result.model_cost_map) + models_count = _swap_in_model_cost_map(reload_result.fetched) self.model_cost_map_loaded_at = current_time await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time) # Adopted last, so neither a failed fetch nor a failed status write is recorded @@ -21011,7 +21012,7 @@ async def reload_model_cost_map( detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.", ) - models_count = _swap_in_model_cost_map(reload_result.model_cost_map) + models_count = _swap_in_model_cost_map(reload_result.fetched) current_time = utc_now() proxy_config.model_cost_map_loaded_at = current_time provenance: Final = get_model_cost_map_provenance() diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..63cdeb32d68 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12284,9 +12284,9 @@ class Router: # tell a resolvable name apart; require usable limits/costs. _azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}" _fallback_entry = litellm.model_cost.get(_azure_fallback_key) - _fallback_resolves = _fallback_entry is not None and ( - (_fallback_entry.get("max_input_tokens") or 0) > 0 - or (_fallback_entry.get("input_cost_per_token") or 0) > 0 + _fallback_resolves = _fallback_entry is not None and any( + isinstance(limit, (int, float)) and limit > 0 + for limit in (_fallback_entry.get("max_input_tokens"), _fallback_entry.get("input_cost_per_token")) ) if _fallback_resolves: verbose_router_logger.debug( diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 7364a1d72f2..183f6f16c1b 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,6 +1,7 @@ from asyncio import Future from collections.abc import Mapping, Sequence -from typing import Never, final +from types import ModuleType +from typing import Literal, Never, final import httpx from pydantic import JsonValue @@ -15,6 +16,72 @@ class ProcessReservedForForking(RuntimeError): ... def trace_encode_error(message: str) -> bytes: ... +RowErrorKind = Literal[ + "json", + "empty", + "below_minimum", + "shrunk", + "not_an_object", + "invalid", + "unknown_block_key", + "missing_provider", + "unknown_timezone", + "alias_shadows_model", + "alias_taken", + "duplicate_price", + "suffix_inside_block", + "tier_range", + "tier_reversed", + "tier_gap", + "mixed_tier_basis", + "not_priced_by_results", + "bands_from_two_sources", + "not_an_off_peak_rate", + "no_off_peak_prices", + "no_off_peak_hours", + "empty_list", + "invalid_date", +] + +@final +class RowError: + key: str + field: str | None + kind: RowErrorKind + rejects: bool + message: str + +@final +class Fetched: + revision: str + etag: str | None + def served(self) -> dict[str, dict[str, JsonValue]]: ... + def document(self) -> dict[str, JsonValue]: ... + def fallback_rules(self) -> list[JsonValue] | None: ... + def errors(self) -> list[RowError]: ... + +@final +class Installed: + count: int + written: int + removed: int + rejected: list[RowError] + +@final +class Unavailable: + reason: str + +@final +class Catalog: + @staticmethod + def bound_to(module: ModuleType, attr: str) -> Catalog: ... + def bundled(self) -> Fetched: ... + @staticmethod + def parse( + body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float + ) -> Fetched | Unavailable: ... + def install(self, fetched: Fetched) -> Installed: ... + @final class NativeClickHouseSpendConfig: def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ... diff --git a/litellm/utils.py b/litellm/utils.py index 1e095966a43..0e3124bdd24 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3830,10 +3830,19 @@ def _registration_attempt( warning_display_name: str | None, final_attempt: bool, ) -> _RegistrationWrite | None: + from litellm.litellm_core_utils.get_model_cost_map import model_cost_install_generation + source: Final = _builtin_cost_map() + generation: Final = model_cost_install_generation() write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name) + if not final_attempt and model_cost_install_generation() != generation: + return None _replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry, value) - return write if final_attempt or litellm.model_cost is source else None + return ( + write + if final_attempt or (litellm.model_cost is source and model_cost_install_generation() == generation) + else None + ) def _register_on_current_map( @@ -6041,6 +6050,10 @@ def get_utc_datetime() -> datetime.datetime: return datetime.datetime.now(datetime.timezone.utc) +def _as_max_tokens(value: object) -> int | None: + return value if isinstance(value, int) else None + + def get_max_tokens(model: str) -> int | None: """ Get the maximum number of output tokens allowed for a given model. @@ -6080,7 +6093,7 @@ def get_max_tokens(model: str) -> int | None: try: if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_output_tokens"] + return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"]) get_llm_provider: Final = litellm_utils.get_llm_provider model, custom_llm_provider, _, _ = get_llm_provider(model=model) if custom_llm_provider == "huggingface": @@ -6088,7 +6101,7 @@ def get_max_tokens(model: str) -> int | None: return max_tokens if model in litellm.model_cost: # check if extracted model is in model_list if "max_output_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_output_tokens"] + return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"]) else: raise Exception() return None @@ -8236,8 +8249,9 @@ def trim_messages( try: if max_tokens is None: # Check if model is valid - if model in litellm.model_cost: - max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"] + trimmed_row: Final = None if model is None else litellm.model_cost.get(model) + max_tokens_for_model: Final = None if trimmed_row is None else trimmed_row.get("max_input_tokens") + if isinstance(max_tokens_for_model, (int, float)): max_tokens = int(max_tokens_for_model * trim_ratio) else: # if user did not specify max (input) tokens @@ -9054,11 +9068,9 @@ def validate_openai_optional_params(stop: str | list[str] | None = None, **kwarg @lru_cache(maxsize=1) def get_bundled_model_cost_map() -> dict[str, Any]: - try: - model_cost_path: Final = resources.files("litellm").joinpath("model_prices_and_context_window_backup.json") - return json.loads(model_cost_path.read_text()) - except Exception: - return {} + from litellm._catalog import CATALOG + + return CATALOG.bundled().document() _get_bundled_model_cost_map = get_bundled_model_cost_map diff --git a/pyproject.toml b/pyproject.toml index 0b3480cab23..95210921ae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -335,6 +335,7 @@ include = [ "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", + { path = "model_prices_and_context_window.json", format = "sdist" }, ] exclude = [ "litellm/proxy/enterprise", @@ -357,6 +358,7 @@ cache-keys = [ { file = "litellm-rust/Cargo.lock" }, { file = "litellm-rust/Cargo.toml" }, { file = "litellm-rust/crates/**/*" }, + { file = "model_prices_and_context_window.json" }, ] constraint-dependencies = [ "tornado>=6.5.8", diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index e9e3f7b8174..0d15890c907 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -1,28 +1,30 @@ +import asyncio +import importlib +import os from collections.abc import Iterator from pathlib import Path from typing import Final +from unittest.mock import patch -import respx - -import asyncio, httpx, importlib, os +import httpx import pytest +import respx import litellm from litellm import CustomLLM from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, - resolve_model_and_provider_for_metadata, get_provider_affinity_header, inferred_provider, is_registered_custom_provider, + resolve_model_and_provider_for_metadata, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.router import LiteLLM_Params from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import patch CUSTOM_PROVIDER: Final = "test-onprem-llm" @@ -634,15 +636,14 @@ def shipped_generalizations(): added on this branch, so these tests install the rule the branch actually ships rather than depending on whatever the live URL returns. """ + from litellm._catalog import CATALOG from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, set_fallback_generalizations, ) - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap previous = list(get_fallback_generalization_rules()) - backup = GetModelCostMap.load_local_model_cost_map() - rules = backup.get("fallback_generalizations", {}).get("rules", []) + rules = CATALOG.bundled().fallback_rules() or [] set_fallback_generalizations(rules) try: yield rules diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index d44169ae28a..1f7553a7207 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -4,16 +4,14 @@ count actual model entries, not reserved meta keys) and the extraction of the ``fallback_generalizations`` block out of the raw map. """ -import asyncio, importlib, importlib.resources as importlib_get_model, json, litellm +import asyncio, hashlib, importlib, json, litellm import os import sys import threading -from pathlib import Path from typing import Final import boto3 import pytest -from pydantic import ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils import get_model_cost_map as get_model_cost_map_module @@ -23,13 +21,11 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_routing_generalization, set_fallback_generalizations, ) +from litellm._catalog import CATALOG, Catalog, Fetched, Unavailable from litellm.litellm_core_utils.get_model_cost_map import ( FALLBACK_GENERALIZATIONS_KEY, GetModelCostMap, - _count_model_entries, - _finalize_model_cost_map, get_model_cost_map_provenance, - git_blob_id, ) @@ -39,75 +35,65 @@ def _load_root_cost_map() -> dict: return json.load(f) +def git_blob_id(body: bytes) -> str: + return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() + + def _bundled_blob_id() -> str: - path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json") + path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json") with open(path, "rb") as f: return git_blob_id(f.read()) -def test_git_blob_id_is_what_git_hash_object_prints(): - assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987" +def _parse(document: dict, min_model_count: int = 50, max_shrink_ratio: float = 0.5) -> Fetched | Unavailable: + return Catalog.parse( + json.dumps(document).encode(), + etag=None, + min_model_count=min_model_count, + max_shrink_ratio=max_shrink_ratio, + ) + + +def test_the_revision_is_what_git_hash_object_prints(): + parsed = Catalog.parse( + b'{"gpt-5.4-mini": {"mode": "chat"}}\n', etag=None, min_model_count=0, max_shrink_ratio=0.0 + ) + assert isinstance(parsed, Fetched) + assert parsed.revision == "18b9a8381e13a3b38a2128f184f631f95829e987" + + +def test_the_bundled_revision_is_the_root_json_blob_id(): + assert CATALOG.bundled().revision == _bundled_blob_id() def _make_models(n: int) -> dict: return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)} -def test_count_model_entries_excludes_reserved_keys(): - m = _make_models(3) - m["sample_spec"] = {"foo": "bar"} - m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - assert _count_model_entries(m) == 3 - - def test_validation_rejects_truly_shrunk_file_even_with_meta_keys(): - """A file with only a handful of real models must be rejected as corrupt, - and the extra meta keys must not inflate the count past the minimum.""" + """Meta keys must not inflate the model count past the minimum""" shrunk = _make_models(5) shrunk["sample_spec"] = {"foo": "bar"} shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": [{"name": "x"}]} - assert ( - GetModelCostMap.validate_model_cost_map( - fetched_map=shrunk, - backup_model_count=2000, - min_model_count=50, - ) - is False - ) + assert isinstance(_parse(shrunk, max_shrink_ratio=0.0), Unavailable) def test_validation_accepts_healthy_file_with_meta_keys(): - healthy = _make_models(2000) + healthy = _make_models(len(CATALOG.bundled().served())) healthy["sample_spec"] = {"foo": "bar"} healthy[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - assert ( - GetModelCostMap.validate_model_cost_map( - fetched_map=healthy, - backup_model_count=2000, - min_model_count=50, - ) - is True - ) + assert isinstance(_parse(healthy), Fetched) -def test_validation_rejects_significant_shrink_vs_backup(): - # 600 real models vs a 2000-model backup is below the 50% shrink threshold. - shrunk = _make_models(600) - shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - assert ( - GetModelCostMap.validate_model_cost_map( - fetched_map=shrunk, - backup_model_count=2000, - min_model_count=50, - max_shrink_ratio=0.5, - ) - is False - ) +def test_validation_rejects_significant_shrink_vs_the_bundled_map(): + shrunk = _make_models(len(CATALOG.bundled().served()) // 3) + + assert isinstance(_parse(shrunk), Unavailable) -def test_finalize_pops_key_and_installs_rules(): +def test_a_loaded_map_installs_its_rules_and_never_serves_the_block(): previous = list(get_fallback_generalization_rules()) try: raw = _make_models(2) @@ -120,21 +106,24 @@ def test_finalize_pops_key_and_installs_rules(): } ] } - finalized = _finalize_model_cost_map(raw) + fetched = _parse(raw, min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(fetched, Fetched) - # The reserved key is removed from the returned model map ... - assert FALLBACK_GENERALIZATIONS_KEY not in finalized - # ... and its rules are installed into the generalizations module. + get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched)) + + assert FALLBACK_GENERALIZATIONS_KEY not in fetched.served() assert match_routing_generalization("widget-9") == "openai" finally: set_fallback_generalizations(previous) -def test_finalize_with_no_block_clears_rules(): +def test_a_loaded_map_with_no_block_clears_rules(): previous = list(get_fallback_generalization_rules()) try: set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]) - _finalize_model_cost_map(_make_models(2)) + fetched = _parse(_make_models(2), min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(fetched, Fetched) + get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched)) assert match_capability_generalizations("x-1") is None finally: set_fallback_generalizations(previous) @@ -144,7 +133,7 @@ def test_shipped_backup_carries_the_claude_routing_rules(): """The bundled backup must ship the Claude routing rules so a fresh install (or an offline fallback) routes unknown Claude models without code changes. Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule.""" - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().document() rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", []) names = [r.get("name") for r in rules] assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids") @@ -166,7 +155,7 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace(): ``bedrockz/anthropic.claude-...`` resolve to bedrock and slip through a ``bedrock/*`` key, so every shipped routing rule must anchor to the start of the name and never match an id carrying an unrecognized namespace prefix.""" - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().document() rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] routing_rules = [r for r in rules if "litellm_provider" in r["model_info"]] @@ -204,7 +193,7 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule carries only its delta; capability unioning stacks it onto the baseline, so the baseline block is never duplicated across rules and no rule needs ``extends``.""" - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().document() rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline") @@ -314,7 +303,7 @@ async def test_refetch_retries_429_honoring_retry_after(): sleeper = _SleepRecorder() result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) - assert len(result.model_cost_map) > 100 + assert len(result.fetched.served()) > 100 assert calls["count"] == 3 assert sleeper.waits == [7.0, 7.0] @@ -384,6 +373,7 @@ async def test_refetch_invalid_json_fails_immediately(): result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "invalid JSON" in result.reason + assert "unusable" in result.reason assert calls["count"] == 1 assert sleeper.waits == [] @@ -395,7 +385,7 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): client, _calls = _mock_client([httpx.Response(200, content=tiny)]) result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) - assert "integrity validation" in result.reason + assert "below 0.5 of reference count" in result.reason @pytest.mark.asyncio @@ -414,7 +404,7 @@ async def test_refetch_respects_local_env_override(monkeypatch, flag): client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)), ) assert isinstance(result, ModelCostMapReloaded) - assert len(result.model_cost_map) > 100 + assert len(result.fetched.served()) > 100 @pytest.mark.asyncio @@ -441,7 +431,7 @@ async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) - monkeypatch.setattr(litellm, "model_cost", result.model_cost_map) + monkeypatch.setattr(litellm, "model_cost", result.fetched.served()) litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False) assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048 assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777 @@ -555,7 +545,7 @@ def test_get_model_cost_map_fetches_s3_catalog(monkeypatch: pytest.MonkeyPatch) for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): monkeypatch.setattr(source_info, name, getattr(source_info, name)) - s3_content: Final = _make_models(GetModelCostMap.get_backup_model_count()) + s3_content: Final = _make_models(len(CATALOG.bundled().served())) s3_client: Final = _FakeS3Client(json.dumps(s3_content).encode()) monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: s3_client) @@ -623,8 +613,10 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_ for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): monkeypatch.setattr(source_info, name, getattr(source_info, name)) + held = litellm.model_cost remote_map = _load_root_cost_map() remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"} + remote_map["gpt-4o"]["input_cost_per_token"] = 0.125 client, calls = _mock_client( [ httpx.ConnectError("connection refused"), @@ -650,7 +642,7 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_ try: assert len(threads) == 1 assert "claude-remote-only-test" not in cost_map - assert cost_map.keys() == _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys() + assert cost_map.keys() == CATALOG.bundled().served().keys() source = get_model_cost_map_source_info() assert source["source"] == "local" assert source["fallback_reason"].startswith("Remote fetch failed:") @@ -661,6 +653,8 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_ assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0 assert calls["count"] == 2 assert "claude-remote-only-test" in litellm.model_cost + assert litellm.model_cost is held + assert held["gpt-4o"]["input_cost_per_token"] == 0.125 assert "claude-remote-only-test" in litellm.anthropic_models assert "my-runtime-model" in litellm.model_cost source = get_model_cost_map_source_info() @@ -768,7 +762,8 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej source = get_model_cost_map_source_info() assert source["source"] == "local" - assert source["fallback_reason"] == "Remote data failed integrity validation" + assert source["fallback_reason"].startswith("Remote fetch failed:") + assert "below minimum" in source["fallback_reason"] assert source["etag"] is None assert source["source_revision"] == _bundled_blob_id() assert source["source_revision"] != git_blob_id(shrunk_body) @@ -862,119 +857,24 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } -@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -class TestCheckIsValidDict: - """Unit tests for _check_is_valid_dict.""" - def test_should_reject_non_dict(self): - """Non-dict should fail.""" - assert GetModelCostMap._check_is_valid_dict("not a dict") is False +@pytest.mark.parametrize( + ("body", "min_model_count", "max_shrink_ratio", "usable"), + [ + pytest.param(b'"not a dict"', 10, 0.5, False, id="not_an_object"), + pytest.param(b"{}", 10, 0.5, False, id="empty"), + pytest.param(b"[1, 2, 3]", 10, 0.5, False, id="list"), + pytest.param(b"null", 10, 0.5, False, id="null"), + pytest.param(json.dumps(_make_models(5)).encode(), 10, 0.0, False, id="below_minimum"), + pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.5, False, id="shrunk"), + pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.0, True, id="no_ratio_floor"), + pytest.param(json.dumps(_make_models(60)).encode(), 61, 0.0, False, id="one_below_minimum"), + ], +) +def test_parse_applies_mains_whole_map_checks(body, min_model_count, max_shrink_ratio, usable): + parsed = Catalog.parse(body, etag=None, min_model_count=min_model_count, max_shrink_ratio=max_shrink_ratio) + assert isinstance(parsed, Fetched) is usable - def test_should_reject_empty_dict(self): - """Empty dict should fail.""" - assert GetModelCostMap._check_is_valid_dict({}) is False - - def test_should_reject_list(self): - """List should fail.""" - assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False - - def test_should_reject_none(self): - """None should fail.""" - assert GetModelCostMap._check_is_valid_dict(None) is False - - def test_should_accept_non_empty_dict(self): - """Non-empty dict should pass.""" - assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True - -@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -class TestCheckModelCountNotReduced: - """Unit tests for _check_model_count_not_reduced.""" - - def test_should_reject_too_few_models(self): - """Fetched map with fewer models than min_model_count should fail.""" - small_map = {f"model-{i}": {} for i in range(5)} - assert ( - GetModelCostMap._check_model_count_not_reduced( - fetched_map=small_map, backup_model_count=0, min_model_count=10 - ) - is False - ) - - def test_should_reject_significant_shrinkage(self): - """Fetched map that shrunk >50% vs backup should fail.""" - fetched = {f"model-{i}": {} for i in range(40)} # 40% of 100 - assert ( - GetModelCostMap._check_model_count_not_reduced( - fetched_map=fetched, backup_model_count=100, min_model_count=10 - ) - is False - ) - - def test_should_accept_when_above_threshold(self): - """Fetched map at 60% of backup (above 50% threshold) should pass.""" - fetched = {f"model-{i}": {} for i in range(60)} - assert ( - GetModelCostMap._check_model_count_not_reduced( - fetched_map=fetched, backup_model_count=100, min_model_count=10 - ) - is True - ) - - def test_should_accept_growth(self): - """Fetched map larger than backup should pass.""" - fetched = {f"model-{i}": {} for i in range(120)} - assert ( - GetModelCostMap._check_model_count_not_reduced( - fetched_map=fetched, backup_model_count=100, min_model_count=10 - ) - is True - ) - - def test_should_accept_with_empty_backup(self): - """When backup is empty, only min_model_count matters.""" - fetched = {f"model-{i}": {} for i in range(15)} - assert ( - GetModelCostMap._check_model_count_not_reduced( - fetched_map=fetched, backup_model_count=0, min_model_count=10 - ) - is True - ) - -@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -class TestValidateModelCostMap: - """Unit tests for validate_model_cost_map (combines both checks).""" - - def test_should_reject_non_dict(self): - """Non-dict should fail at check 1.""" - assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_model_count=0) is False - - def test_should_reject_empty_map(self): - """Empty dict should fail at check 1.""" - assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_model_count=0) is False - - def test_should_reject_significant_shrinkage(self): - """Should fail at check 2 (shrinkage).""" - fetched = {f"model-{i}": {} for i in range(40)} - assert ( - GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10) - is False - ) - - def test_should_accept_valid_map(self): - """Should pass both checks.""" - fetched = {f"model-{i}": {} for i in range(120)} - assert ( - GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10) - is True - ) - - def test_should_accept_equal_size_map(self): - """Equal size should pass both checks.""" - fetched = {f"model-{i}": {} for i in range(100)} - assert ( - GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10) - is True - ) def _fetch_with_single_outcome(monkeypatch, outcome): from litellm.litellm_core_utils import get_model_cost_map as module @@ -989,7 +889,7 @@ def _fetch_with_single_outcome(monkeypatch, outcome): def _backup_keys(): - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys() + return CATALOG.bundled().served().keys() @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @@ -1028,7 +928,7 @@ class TestGetModelCostMapFallback: assert fetches == 1 assert result.keys() == _backup_keys() assert source["source"] == "local" - assert source["fallback_reason"] == f"Remote fetch failed: expected a JSON object from {_URL}, got list" + assert source["fallback_reason"].startswith(f"Remote fetch failed: model cost map from {_URL} is unusable") def test_should_fallback_when_fetched_map_shrinks_dramatically(self, monkeypatch): """When upstream returns far fewer models than backup, should fall back.""" @@ -1039,7 +939,8 @@ class TestGetModelCostMapFallback: assert fetches == 1 assert result.keys() == _backup_keys() - assert source == {**source, "source": "local", "fallback_reason": "Remote data failed integrity validation"} + assert source["source"] == "local" + assert "below minimum" in source["fallback_reason"] def test_should_use_local_map_when_env_var_set(self): """LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely.""" @@ -1056,14 +957,14 @@ class TestBackupModelCostMapExists: """Validates the local backup file is always present and valid.""" def test_should_have_backup_file(self): - """The backup model cost map must exist and be loadable.""" - backup = GetModelCostMap.load_local_model_cost_map() + """The bundled model cost map must be embedded and loadable.""" + backup = CATALOG.bundled().served() assert isinstance(backup, dict) assert len(backup) > 0, "Backup model cost map is empty" def test_should_have_minimum_models_in_backup(self): """The backup must contain a reasonable number of models.""" - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().served() assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100" @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @@ -1241,26 +1142,156 @@ def test_get_model_cost_map(): except Exception as e: pytest.fail(f"An exception occurred: {e}") -@pytest.mark.usefixtures( - "_vcr_outcome_gate_local_testing", - "isolate_litellm_state", - "setup_and_teardown_local_testing", -) -def test_get_backup_model_cost_map(): - with importlib_get_model.open_text("litellm", "model_prices_and_context_window_backup.json") as f: - print("inside backup") - content = json.load(f) - print("content", content) -@pytest.mark.parametrize("backup", [b'["only-in-the-backup"]', b'"only-in-the-backup"', b"null"]) -def test_a_bundled_backup_that_is_not_a_json_object_is_rejected_without_quoting_it( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, backup: bytes -): - (tmp_path / "model_prices_and_context_window_backup.json").write_bytes(backup) - monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path) +# --------------------------------------------------------------------------- +# Install through the Rust catalog (decisions 1, 15, 23 and 25) +# --------------------------------------------------------------------------- - with pytest.raises(ValidationError) as rejected: - GetModelCostMap.load_local_model_cost_map_with_revision() +import logging +import types - assert "only-in-the-backup" not in str(rejected.value) + +@functools.lru_cache(maxsize=1) +def _bundled_document() -> dict: + return CATALOG.bundled().document() + + +def _remote(edit) -> Fetched: + document = json.loads(json.dumps(_bundled_document())) + edit(document) + parsed = Catalog.parse(json.dumps(document).encode(), etag=None, min_model_count=50, max_shrink_ratio=0.5) + assert isinstance(parsed, Fetched) + return parsed + + +def _bound_catalog() -> tuple[types.ModuleType, Catalog]: + module = types.ModuleType("fake_litellm") + module.model_cost = {} + catalog = Catalog.bound_to(module, "model_cost") + catalog.install(catalog.bundled()) + return module, catalog + + +def test_sample_spec_is_not_served_but_stays_in_the_bundled_document(): + assert "sample_spec" not in litellm.model_cost + assert "sample_spec" in CATALOG.bundled().document() + + +def test_a_fetched_row_that_fails_validation_is_replaced_by_bundled_or_left_out(caplog): + module, catalog = _bound_catalog() + bundled_gpt_4o = dict(module.model_cost["gpt-4o"]) + + def edit(document): + document["gpt-4o"]["input_cost_per_token"] = -1 + document["acme/new"] = {"litellm_provider": "acme", "input_cost_per_token": "free"} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + installed = get_model_cost_map_module._install( # pyright: ignore[reportPrivateUsage] # only the install step logs rejected rows + _remote(edit), catalog + ) + + assert module.model_cost["gpt-4o"] == bundled_gpt_4o + assert "acme/new" not in module.model_cost + assert {error.key for error in installed.rejected} == {"gpt-4o", "acme/new"} + warnings = [record for record in caplog.records if "failed validation" in record.getMessage()] + assert len(warnings) == 1 + assert "gpt-4o" in warnings[0].getMessage() and "acme/new" in warnings[0].getMessage() + + +def test_a_field_the_fetched_row_lacks_is_served_from_bundled(): + module, catalog = _bound_catalog() + bundled_max_output_tokens = module.model_cost["gpt-4o"]["max_output_tokens"] + + catalog.install(_remote(lambda document: document["gpt-4o"].pop("max_output_tokens"))) + + assert module.model_cost["gpt-4o"]["max_output_tokens"] == bundled_max_output_tokens + + +def test_a_key_the_fetched_map_lacks_stays_until_the_map_writes_it_as_null(): + module, catalog = _bound_catalog() + + catalog.install(_remote(lambda document: document.pop("gpt-4o"))) + assert "gpt-4o" in module.model_cost + + catalog.install(_remote(lambda document: document.update({"gpt-4o": None}))) + assert "gpt-4o" not in module.model_cost + + +def test_a_null_field_in_the_fetched_map_removes_it(): + module, catalog = _bound_catalog() + + catalog.install(_remote(lambda document: document["gpt-4o"].update({"max_output_tokens": None}))) + + assert "max_output_tokens" not in module.model_cost["gpt-4o"] + + +def _direct_writes(model_cost: dict) -> None: + model_cost["gpt-4o"]["input_cost_per_token"] = 1.0 + model_cost["my-own-model"] = {"litellm_provider": "openai", "input_cost_per_token": 3.0} + model_cost["gpt-4o-mini"] = {"litellm_provider": "openai", "input_cost_per_token": 4.0} + del model_cost["gpt-4.1"] + + +def _changed_upstream(document: dict) -> None: + document["gpt-4o"]["input_cost_per_token"] = 9.0 + document["gpt-4o-mini"]["input_cost_per_token"] = 9.0 + document["gpt-4o-mini"]["max_input_tokens"] = 1 + + +def test_direct_writes_survive_a_reload(): + module, catalog = _bound_catalog() + held = module.model_cost + _direct_writes(module.model_cost) + + catalog.install(_remote(_changed_upstream)) + + assert module.model_cost is held + assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 1.0 + assert module.model_cost["my-own-model"]["input_cost_per_token"] == 3.0 + assert module.model_cost["gpt-4o-mini"]["input_cost_per_token"] == 4.0 + assert "max_input_tokens" not in module.model_cost["gpt-4o-mini"] + assert "gpt-4.1" not in module.model_cost + + +def test_rebinding_the_dict_then_reloading_drops_direct_writes(): + module, catalog = _bound_catalog() + _direct_writes(module.model_cost) + module.model_cost = dict(module.model_cost) + + catalog.install(_remote(_changed_upstream)) + + assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 9.0 + assert "my-own-model" not in module.model_cost + assert module.model_cost["gpt-4o-mini"]["max_input_tokens"] == 1 + assert "gpt-4.1" in module.model_cost + + +def test_install_keeps_a_row_object_whose_fields_did_not_change(): + module, catalog = _bound_catalog() + row = module.model_cost["gpt-4o"] + + catalog.install(_remote(lambda document: document["gpt-4o-mini"].update({"input_cost_per_token": 9.0}))) + + assert module.model_cost["gpt-4o"] is row + + +def test_install_refuses_a_bound_attribute_that_is_not_a_dict(): + module, catalog = _bound_catalog() + module.model_cost = types.MappingProxyType({}) + + with pytest.raises(TypeError): + catalog.install(catalog.bundled()) + + +def test_the_local_map_is_the_embedded_one_not_the_backup_file(monkeypatch, tmp_path): + (tmp_path / "model_prices_and_context_window_backup.json").write_text( + json.dumps({"only-in-the-backup": {"litellm_provider": "openai"}}) + ) + monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path, raising=False) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + cost_map = get_model_cost_map(url=_URL) + + assert "only-in-the-backup" not in cost_map + assert cost_map.keys() == CATALOG.bundled().served().keys() diff --git a/tests/unit/litellm_core_utils/test_tokenizer.py b/tests/unit/litellm_core_utils/test_tokenizer.py index 61e797d7166..8a928ae3a85 100644 --- a/tests/unit/litellm_core_utils/test_tokenizer.py +++ b/tests/unit/litellm_core_utils/test_tokenizer.py @@ -241,21 +241,15 @@ print("compatible") @pytest.mark.parametrize("rust", (None, "0", "1")) -def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None: +def test_tokenization_stays_offline(tmp_path: Path, rust: str | None) -> None: script: Final = """ -import importlib.abc import sys sys.path.insert(0, sys.argv[1]) +import litellm def reject_network(event, args): if event == "socket.connect": raise AssertionError("tokenizer attempted a network connection") sys.addaudithook(reject_network) -class Block(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname == "litellm.rust_bridge._native": - raise ImportError("native extension is unavailable") -sys.meta_path.insert(0, Block()) -import litellm from litellm.rust_bridge.tokenizer import get_encoding import tiktoken from tokenizers import Tokenizer diff --git a/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index fdd21c87732..94ceca04b2d 100644 --- a/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -6,7 +6,7 @@ import httpx import pytest import litellm -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm._catalog import CATALOG from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils from litellm.llms.azure.azure import AzureChatCompletion from litellm.llms.azure.image_generation import get_azure_image_generation_config @@ -20,7 +20,7 @@ from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_par @pytest.fixture(autouse=True) def use_local_model_cost_map(monkeypatch): - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served()) litellm.get_model_info.cache_clear() _invalidate_model_cost_lowercase_map() yield diff --git a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py index 664dbf87fe9..4d704096a5d 100644 --- a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py +++ b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py @@ -8,10 +8,10 @@ the sharded CI (coverage collection runs against this tree). import json import os +from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from unittest.mock import AsyncMock, patch, MagicMock import litellm from litellm.llms.base_llm.search.transformation import SearchResponse @@ -632,10 +632,10 @@ class TestAgentCoreSearchEdgeCases: def test_search_cost_lookup_is_mapped(self, monkeypatch): """Assert against the map in this checkout: the remote cost map litellm loads by default only carries providers already released.""" - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + from litellm._catalog import CATALOG from litellm.search.cost_calculator import search_provider_cost_per_query - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served()) price_per_query = litellm.model_cost["agentcore/search"]["input_cost_per_query"] assert price_per_query > 0 assert search_provider_cost_per_query( diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 7bde1805e18..75ca6fb16a4 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,9 +1,16 @@ -import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +import asyncio +import importlib +from collections.abc import Iterator from typing import Final +from unittest.mock import Mock, patch +import pytest -from litellm.llms.bedrock.common_utils import( +import litellm +import litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map +from litellm import completion +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, bedrock_native_output_is_opt_in, @@ -16,11 +23,8 @@ from litellm.llms.bedrock.common_utils import( strip_bedrock_throughput_suffix, without_bedrock_native_output_opt_in, ) -from collections.abc import Iterator -from litellm import completion from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import Mock, patch # --------------------------------------------------------------------------- # # get_bedrock_response_stream_shape lazy-load tests # @@ -33,10 +37,8 @@ def _reset_bedrock_response_stream_shape_cache(): import litellm.llms.bedrock.common_utils as mod mod.get_bedrock_response_stream_shape.cache_clear() - mod._get_local_model_cost_map.cache_clear() yield mod.get_bedrock_response_stream_shape.cache_clear() - mod._get_local_model_cost_map.cache_clear() def test_bedrock_response_stream_shape_lazy_loads_once(): @@ -342,9 +344,9 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( model, expected_ceiling ): - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + from litellm._catalog import CATALOG - model_info = GetModelCostMap.load_local_model_cost_map()[model] + model_info = CATALOG.bundled().served()[model] assert model_info["bedrock_output_config_effort_ceiling"] == expected_ceiling @@ -2034,6 +2036,7 @@ class TestBedrockGovCloudSupport: @pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"]) def test_event_decoder_reports_only_missing_aws_dependency(missing): from unittest.mock import patch + from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase failure = ModuleNotFoundError("missing dependency", name=missing) diff --git a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py index ee2dd627563..a775e9b8f1d 100644 --- a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py @@ -7,12 +7,11 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations. from unittest.mock import patch - import pytest import litellm import litellm.utils -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm._catalog import CATALOG from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig @@ -681,7 +680,7 @@ class TestKimiK26ModelRegistry: @pytest.fixture(autouse=True) def model_cost_map(self): """Load directly from the bundled backup so tests don't depend on remote fetch.""" - return GetModelCostMap.load_local_model_cost_map() + return CATALOG.bundled().served() class TestMoonshotResponseSchemaSupport: @@ -703,7 +702,7 @@ class TestMoonshotResponseSchemaSupport: @pytest.fixture(autouse=True) def model_cost_map(self): - return GetModelCostMap.load_local_model_cost_map() + return CATALOG.bundled().served() class TestMoonshotReasoningEffort: @@ -713,7 +712,7 @@ class TestMoonshotReasoningEffort: @pytest.fixture(autouse=True) def force_local_model_cost(self, monkeypatch): - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served()) @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"]) def test_reasoning_model_supports_reasoning_effort(self, model): diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index 92cfb6a71e5..878e42c52b3 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -64,6 +64,15 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( _STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",) +def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": + from litellm._catalog import Catalog, Fetched + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + + parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(parsed, Fetched) + return ModelCostMapReloaded(fetched=parsed) + + class _RecordingRedisClient(Redis): def __init__(self) -> None: self.published: List[Tuple[str, str]] = [] @@ -860,7 +869,7 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None: with patch( "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new=AsyncMock( - return_value=ModelCostMapReloaded(model_cost_map={"gpt-5.2": {"input_cost_per_token": 0.001}}) + return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}}) ), ): await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py index 31f69d686d4..4f4dac169cb 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py @@ -32,6 +32,15 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context # --------------------------------------------------------------------------- +def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": + from litellm._catalog import Catalog, Fetched + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + + parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(parsed, Fetched) + return ModelCostMapReloaded(fetched=parsed) + + def _attach_litellm_config(mock_prisma): """Attach a litellm_config table mock (not in conftest's _PRISMA_TABLES).""" table = MagicMock() @@ -55,7 +64,7 @@ def _attach_litellm_config(mock_prisma): def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): """Admin can trigger a manual reload; handler returns model count + status.""" - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + from litellm._catalog import CATALOG from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -65,7 +74,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fake_cost_map)), + AsyncMock(return_value=_reloaded(fake_cost_map)), ) monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) monkeypatch.setattr("litellm.model_cost", {}, raising=False) @@ -83,11 +92,12 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.post("/reload/model_cost_map") assert response.status_code == 200 + served = len(CATALOG.bundled().served().keys() | fake_cost_map.keys()) body = normalize(response.json(), volatile=_VOLATILE) assert body == { - "message": "Price data reloaded successfully! 2 models updated.", + "message": f"Price data reloaded successfully! {served} models updated.", "status": "success", - "models_count": 2, + "models_count": served, "timestamp": "", **get_model_cost_map_provenance(), } @@ -103,7 +113,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every import httpx import litellm - from litellm.litellm_core_utils.get_model_cost_map import _finalize_model_cost_map, git_blob_id + import hashlib from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -111,7 +121,10 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) body = _ROOT_COST_MAP.read_bytes() - expected = {"source_revision": git_blob_id(body), "etag": _SERVED_ETAG} + expected = { + "source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(), + "etag": _SERVED_ETAG, + } served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body) monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", @@ -142,7 +155,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every assert {key: status_response.json()[key] for key in expected} == expected assert public_response.status_code == 200 assert "gpt-4o" in public_response.json() - assert reload_body["models_count"] == len(_finalize_model_cost_map(json.loads(body))) + assert reload_body["models_count"] == len(litellm.model_cost) def test_reload_model_cost_map_fetch_failure_502_keeps_map( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 55d1587669a..947a773e4ef 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -80,6 +80,15 @@ example_embedding_result = { } +def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": + from litellm._catalog import Catalog, Fetched + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + + parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(parsed, Fetched) + return ModelCostMapReloaded(fetched=parsed) + + def mock_patch_aembedding(): return mock.patch( "litellm.proxy.proxy_server.llm_router.aembedding", @@ -4551,7 +4560,7 @@ class TestPriceDataReloadAPI: with patch( "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new=AsyncMock( - return_value=ModelCostMapReloaded(model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}) + return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}) ), ): # Mock the database connection @@ -4569,8 +4578,11 @@ class TestPriceDataReloadAPI: assert "timestamp" in data assert "models_count" in data # The new implementation immediately reloads and returns the count - assert "Price data reloaded successfully! 1 models updated." in data["message"] - assert data["models_count"] == 1 + from litellm._catalog import CATALOG + + served = len(CATALOG.bundled().served()) + assert f"Price data reloaded successfully! {served} models updated." in data["message"] + assert data["models_count"] == served finally: # Restore the full model cost map so subsequent tests are not affected litellm.model_cost = original_model_cost @@ -4822,7 +4834,7 @@ class TestPriceDataReloadIntegration: try: with patch( "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=mock_cost_map)), + new=AsyncMock(return_value=_reloaded(mock_cost_map)), ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: @@ -4895,13 +4907,12 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) - assert litellm.model_cost["gpt-3.5-turbo"] == {"input_cost_per_token": 0.001} + assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.001 assert proxy_config.model_cost_map_loaded_at == frozen_now assert mock_prisma.db.litellm_config.update_many.call_args[1] == { "data": {"last_run_at": frozen_now}, @@ -4980,13 +4991,12 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4-test": {"input_cost_per_token": 0.5}} + mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) - assert litellm.model_cost["gpt-4-test"] == {"input_cost_per_token": 0.5} + assert litellm.model_cost["gpt-4-test"]["input_cost_per_token"] == 0.5 assert proxy_config.model_cost_map_loaded_at == frozen_now assert mock_prisma.db.litellm_config.update_many.call_args[1]["data"] == {"last_run_at": frozen_now} @@ -5025,8 +5035,7 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} ) for _ in range(3): @@ -5069,8 +5078,7 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5105,8 +5113,7 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5147,8 +5154,7 @@ class TestPriceDataReloadIntegration: ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4": {"input_cost_per_token": 0.1}} + mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5236,8 +5242,7 @@ class TestPriceDataReloadIntegration: patch( "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new=AsyncMock( - return_value=ModelCostMapReloaded( - model_cost_map={"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}} + return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}} ) ), ), @@ -5271,9 +5276,9 @@ class TestPriceDataReloadIntegration: model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} ) - models_count = _swap_in_model_cost_map({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}) + models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched) - assert models_count == 1 + assert models_count == len(litellm.model_cost) - 1 assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 finally: litellm.model_cost = original_model_cost @@ -5311,8 +5316,7 @@ class TestPriceDataReloadIntegration: patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = ModelCostMapReloaded( - model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} ) mock_prisma.db.litellm_config.upsert = AsyncMock( return_value=_reload_schedule_row({}, reload_revision=9) diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index e30fd2eb6f8..4fc69812d5d 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -8,6 +8,7 @@ import pytest import respx import litellm +from litellm._catalog import CATALOG from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.litellm_logging import Logging from litellm.search.cost_calculator import search_provider_cost_per_query @@ -29,7 +30,7 @@ SEARCH_MODE_PRICES: Final = { @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served()) @pytest.fixture diff --git a/tests/unit/test_claude_fable_5_config.py b/tests/unit/test_claude_fable_5_config.py index 7ec76742bb8..12d9d7e68ff 100644 --- a/tests/unit/test_claude_fable_5_config.py +++ b/tests/unit/test_claude_fable_5_config.py @@ -14,8 +14,8 @@ import os import pytest +from litellm._catalog import CATALOG from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -64,7 +64,7 @@ FABLE_5_1_VARIANTS = ( def test_fable_5_1_present_in_bundled_backup(): - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().served() root = _load_root_cost_map() for model_name in FABLE_5_1_VARIANTS: assert model_name in backup, f"Missing from backup cost map: {model_name}" diff --git a/tests/unit/test_claude_haiku_5_5_config.py b/tests/unit/test_claude_haiku_5_5_config.py index 99c7d167df4..6c747e4be99 100644 --- a/tests/unit/test_claude_haiku_5_5_config.py +++ b/tests/unit/test_claude_haiku_5_5_config.py @@ -16,7 +16,7 @@ from typing import Final, cast import pytest import litellm -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm._catalog import CATALOG from litellm.llms.anthropic.common_utils import AnthropicModelInfo REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "../..") @@ -74,7 +74,7 @@ def test_haiku_5_5_rows_allow_disabling_thinking_and_forced_tools( model_name: str, ) -> None: root: Final = _load_root_cost_map() - backup: Final = GetModelCostMap.load_local_model_cost_map() + backup: Final = CATALOG.bundled().served() assert model_name in root row: Final = root[model_name] # https://platform.claude.com/docs/en/models/haiku-5-5/whats-new-haiku-5-5 (2026-10-07): diff --git a/tests/unit/test_claude_opus_5_config.py b/tests/unit/test_claude_opus_5_config.py index bac4beb494c..487ffc081fb 100644 --- a/tests/unit/test_claude_opus_5_config.py +++ b/tests/unit/test_claude_opus_5_config.py @@ -19,8 +19,8 @@ import os import pytest +from litellm._catalog import CATALOG from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -82,7 +82,7 @@ OPUS_5_5_VARIANTS = ( @pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS) def test_opus_5_5_present_in_bundled_backup(model_name): - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().served() root = _load_root_cost_map() assert model_name in backup assert model_name in root diff --git a/tests/unit/test_claude_sonnet_5_config.py b/tests/unit/test_claude_sonnet_5_config.py index eb040ceae1b..3439b81a8ad 100644 --- a/tests/unit/test_claude_sonnet_5_config.py +++ b/tests/unit/test_claude_sonnet_5_config.py @@ -19,8 +19,8 @@ import os import pytest +from litellm._catalog import CATALOG from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -60,7 +60,7 @@ SONNET_5_5_VARIANTS = ( @pytest.mark.parametrize("model_name", SONNET_5_5_VARIANTS) def test_sonnet_5_5_present_in_bundled_backup(model_name): - backup = GetModelCostMap.load_local_model_cost_map() + backup = CATALOG.bundled().served() root = _load_root_cost_map() assert model_name in backup assert model_name in root diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 578d5482822..cfb24580fac 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -186,16 +186,32 @@ def test_bot_may_only_touch_the_cost_map_files() -> None: ) +_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it" +_REMOVED_VISION: Final = ( + "openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it" +) + + def test_bot_may_not_remove_models() -> None: head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"}) - assert _failures(head, bot=False) == () - assert _failures(head) == ("bot PRs may not remove models: openrouter/b",) + assert _failures(head, bot=False) == (_REMOVED_B,) + assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b") def test_bot_may_not_remove_fields() -> None: head = _snapshot({**BASE_MAP, "openrouter/a": _entry()}) + assert _failures(head, bot=False) == (_REMOVED_VISION,) + assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision") + + +def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None: + head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None}) + assert _failures(head, bot=False) == () + + +def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None: + head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}}) assert _failures(head, bot=False) == () - assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",) def test_bot_may_not_change_special_root_keys() -> None: @@ -244,7 +260,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co ("head_ref", "expected_code", "expected_line"), [ (BOT_REF, 1, "- bot PRs may not remove models: openrouter/b"), - ("litellm_fix_pricing", 0, "cost map guard passed (human PR, file checks only)"), + ("litellm_fix_pricing", 1, f"- {_REMOVED_B}"), ], ) def test_main_reads_both_revisions_from_git( diff --git a/tests/unit/test_model_cost_aliases.py b/tests/unit/test_model_cost_aliases.py index f9f92a85cb2..8d63ae658bb 100644 --- a/tests/unit/test_model_cost_aliases.py +++ b/tests/unit/test_model_cost_aliases.py @@ -1,14 +1,28 @@ """ Tests for the ``aliases`` feature in the model cost map. -The ``_expand_model_aliases`` function processes ``aliases`` lists from model -entries, creating shared dict references for alias entries at load time. +The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry """ -from unittest.mock import patch +import json -from litellm import verbose_logger -from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases +from litellm._catalog import Catalog, Fetched, Unavailable + + +def _parsed(model_cost: dict) -> Fetched | Unavailable: + return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) + + +def _expand(model_cost: dict) -> dict: + parsed = _parsed(model_cost) + assert isinstance(parsed, Fetched) + return parsed.served() + + +def _error_kinds(model_cost: dict) -> list[str]: + parsed = _parsed(model_cost) + assert isinstance(parsed, Fetched) + return [error.kind for error in parsed.errors()] # --------------------------------------------------------------------------- @@ -29,7 +43,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert "my-model-20250101" in result assert result["my-model-20250101"]["input_cost_per_token"] == 1e-06 @@ -45,7 +59,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert "provider/model-v1" in result assert "provider/model-v2" in result @@ -60,7 +74,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert result["alias-model"] is result["canonical-model"] @@ -74,7 +88,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert "aliases" not in result["my-model"] assert "aliases" not in result["my-model-alias"] @@ -88,7 +102,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert "plain-model" in result assert result["plain-model"]["input_cost_per_token"] == 3e-06 @@ -104,7 +118,7 @@ class TestExpandModelAliases: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert len(result) == 1 assert "model-a" in result @@ -134,15 +148,11 @@ class TestAliasConflicts: "mode": "chat", }, } - with patch.object(verbose_logger, "warning") as mock_warn: - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) # The canonical "model-dated" entry is preserved, not overwritten assert "model-dated" in result - # Verify a warning about the alias conflict was logged - mock_warn.assert_called() - warning_messages = " ".join(str(c) for c in mock_warn.call_args_list) - assert "alias conflict" in warning_messages.lower() + assert "alias_shadows_model" in _error_kinds(model_cost) def test_duplicate_alias_across_entries(self): """Same alias claimed by two different entries: second one is skipped.""" @@ -160,16 +170,12 @@ class TestAliasConflicts: "mode": "chat", }, } - with patch.object(verbose_logger, "warning") as mock_warn: - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) # "shared-alias" should point to model-a (first one wins) assert "shared-alias" in result assert result["shared-alias"]["input_cost_per_token"] == 1e-06 - # Verify a warning about the alias conflict was logged - mock_warn.assert_called() - warning_messages = " ".join(str(c) for c in mock_warn.call_args_list) - assert "alias conflict" in warning_messages.lower() + assert "alias_taken" in _error_kinds(model_cost) def test_canonical_entry_not_overwritten_by_alias(self): """An alias must never overwrite an existing canonical entry's data.""" @@ -187,7 +193,7 @@ class TestAliasConflicts: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) # Original entry must be preserved assert result["existing-model"]["input_cost_per_token"] == original_cost @@ -211,7 +217,7 @@ class TestAliasIntegration: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) # Mutate via alias result["alias"]["input_cost_per_token"] = 999 @@ -232,7 +238,7 @@ class TestAliasIntegration: "mode": "chat", }, } - result = _expand_model_aliases(model_cost) + result = _expand(model_cost) assert len(result) == 4 # 2 canonical + 2 aliases assert "alias-1" in result @@ -240,6 +246,5 @@ class TestAliasIntegration: assert "plain-model" in result assert "model-with-alias" in result - def test_expand_on_empty_dict(self): - """Expanding an empty dict returns an empty dict.""" - assert _expand_model_aliases({}) == {} + def test_an_empty_map_is_unavailable(self): + assert isinstance(_parsed({}), Unavailable) diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 1d9c2e87100..49fe73d9c64 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1384,13 +1384,6 @@ ALIAS_CASES: Final = ( "is_non_openai_azure_model", False, ), - ( - "litellm.litellm_core_utils.get_model_cost_map", - "GetModelCostMap", - "_get_backup_model_count", - "get_backup_model_count", - False, - ), ( "litellm.litellm_core_utils.health_check_helpers", "HealthCheckHelpers", diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..402c0ca05fd 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -11,6 +11,7 @@ calculations for DB-sourced models with prompt caching pricing. import copy import itertools +import json import os from collections.abc import Callable, Mapping from typing import Final, NoReturn @@ -19,11 +20,18 @@ import pytest import litellm -from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map +from litellm._catalog import Catalog, Fetched +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map from litellm.main import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map +def _fetched(cost_map: Mapping[str, object]) -> Fetched: + parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) + assert isinstance(parsed, Fetched) + return parsed + + def _snapshot_model_cost_entries(keys): return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys} @@ -1038,11 +1046,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost}, } real_lookup: Final = litellm.utils.get_model_info + reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object: info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider) - if litellm.model_cost is not reloaded_cost_map: - adopt_model_cost_map(reloaded_cost_map) + if not reloads: + reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map))) return info with pytest.MonkeyPatch.context() as lookup_patch: @@ -1056,52 +1065,6 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk _invalidate_model_cost_lowercase_map() -def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model" - deployment_key: Final = f"bedrock/{catalog_key}" - catalog_row: Final = { - "litellm_provider": "bedrock_converse", - "mode": "chat", - "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 1.65e-05, - } - reloaded_input_cost: Final = 6.6e-06 - registered_output_cost: Final = 2e-05 - builtin_cost_map: Final = litellm.get_model_cost_map(url="") - reloaded_cost_map: Final = { - **builtin_cost_map, - catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost}, - } - - class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]): - def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping - if litellm.model_cost is self: - adopt_model_cost_map(reloaded_cost_map) - super().update(other) - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr( - litellm, - "model_cost", - CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}), - ) - _invalidate_model_cost_lowercase_map() - - litellm.register_model( - {deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}}, - persist_across_reloads=False, - ) - - try: - assert litellm.model_cost is reloaded_cost_map - assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost - assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost - finally: - _invalidate_model_cost_lowercase_map() - - def test_register_model_leaves_the_entry_a_lookup_is_walking_intact(monkeypatch: pytest.MonkeyPatch) -> None: catalog_key: Final = "lit-5853-live-entry-model" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -1139,21 +1102,14 @@ _ALIAS_KEY: Final = "lit-5853-alias-model" def _install_aliased_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + aliased: Final = AliasedCostEntry( + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06}, + (_CANONICAL_KEY, _ALIAS_KEY), + ) monkeypatch.setattr( litellm, "model_cost", - _expand_model_aliases( - { - **litellm.get_model_cost_map(url=""), - _CANONICAL_KEY: { - "aliases": [_ALIAS_KEY], - "litellm_provider": "openai", - "mode": "chat", - "input_cost_per_token": 2e-06, - "output_cost_per_token": 8e-06, - }, - } - ), + {**litellm.get_model_cost_map(url=""), _CANONICAL_KEY: aliased, _ALIAS_KEY: aliased}, ) _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d9b655a0aff..5d56d734159 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20337,9 +20337,9 @@ class TestTierParamsTheTargetAccepts: @pytest.fixture(autouse=True) def force_local_model_cost(self, monkeypatch): - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + from litellm._catalog import CATALOG - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served()) @staticmethod def _router(model: str) -> litellm.Router: