mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(model-cost): load litellm.model_cost through the Rust catalog
The Rust catalog parses, validates and installs the cost map into the dict bound to litellm.model_cost. Fetched rows patch the bundled rows, rejected rows fall back to bundled, and fields the user wrote stay theirs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
21acd3ce29
commit
18665e09be
46 changed files with 1665 additions and 754 deletions
|
|
@ -5,6 +5,9 @@ checks: the files parse and the backup copy matches the root file. A pull reques
|
|||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
|
||||
Installed wheels layer a fetched map over their bundled one, so a key or a price or ``supports_*`` field the map
|
||||
leaves out is still served from the wheel. Every pull request that drops one must write it as ``null`` instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -91,6 +94,35 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
|
|||
return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
|
||||
|
||||
|
||||
def _retired_by_null(field: str) -> bool:
|
||||
return "cost" in field or field in ("tiered_pricing", "off_peak_pricing") or field.startswith("supports_")
|
||||
|
||||
|
||||
def _silent_removal_failures(base: Snapshot, head_map: CostMap) -> tuple[str, ...]:
|
||||
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
|
||||
if isinstance(base_map, str):
|
||||
return ()
|
||||
head_entries: Final = _entries(head_map)
|
||||
removed_fields: Final = tuple(
|
||||
f"{key}.{field}"
|
||||
for key, entry in _entries(base_map).items()
|
||||
if key in head_entries
|
||||
for field in entry
|
||||
if _retired_by_null(field) and field not in head_entries[key]
|
||||
)
|
||||
return (
|
||||
*(
|
||||
f"{key} is removed without a null; write it as null so older wheels stop serving it"
|
||||
for key in base_map
|
||||
if key not in head_map and key not in SPECIAL_ROOT_KEYS
|
||||
),
|
||||
*(
|
||||
f"{ref} is removed without a null; write it as null so older wheels stop serving it"
|
||||
for ref in removed_fields
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
|
||||
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
|
||||
if isinstance(base_map, str):
|
||||
|
|
@ -147,6 +179,7 @@ def guard_failures(
|
|||
*_file_failures(
|
||||
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
|
||||
),
|
||||
*_silent_removal_failures(base, head_map),
|
||||
*(_bot_failures(base, head_map, changed_files) if bot else ()),
|
||||
)
|
||||
|
||||
|
|
|
|||
3
litellm-rust/Cargo.lock
generated
3
litellm-rust/Cargo.lock
generated
|
|
@ -4268,6 +4268,7 @@ dependencies = [
|
|||
"criterion",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
"indexmap 2.14.0",
|
||||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
"litellm-cache",
|
||||
|
|
@ -4287,6 +4288,7 @@ dependencies = [
|
|||
"litellm-inference-transcription",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-model-catalog",
|
||||
"litellm-router-types",
|
||||
"litellm-secrets",
|
||||
"litellm-secrets-aws",
|
||||
|
|
@ -4304,6 +4306,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sha1 0.10.7",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ pub struct Catalog {
|
|||
aliases: IndexMap<String, String>,
|
||||
lowercase_keys: HashMap<String, String>,
|
||||
fallback_generalizations: Option<FallbackGeneralizations>,
|
||||
removals: Vec<String>,
|
||||
}
|
||||
|
||||
impl IntegrityLimits {
|
||||
|
|
@ -96,7 +97,17 @@ impl Catalog {
|
|||
let mut sections = Vec::new();
|
||||
let mut entries = IndexMap::with_capacity(document.len());
|
||||
let mut alias_lists = Vec::new();
|
||||
let mut removals = Vec::new();
|
||||
for (index, (name, value)) in document.into_iter().enumerate() {
|
||||
if value.is_null() && !is_section(&name) {
|
||||
removals.push(name.clone());
|
||||
sections.push(Section {
|
||||
index,
|
||||
key: name,
|
||||
value,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let fields = match value {
|
||||
Value::Object(fields) if !is_section(&name) => fields,
|
||||
value => {
|
||||
|
|
@ -169,6 +180,7 @@ impl Catalog {
|
|||
aliases,
|
||||
lowercase_keys,
|
||||
fallback_generalizations,
|
||||
removals,
|
||||
},
|
||||
errors: problems,
|
||||
})
|
||||
|
|
@ -280,6 +292,45 @@ impl Catalog {
|
|||
(catalog, removed_errors)
|
||||
}
|
||||
|
||||
/// Rows replaced by the same key's row in `bundled` when it has one, and left out otherwise (decision 15)
|
||||
pub fn with_rejected_replaced<'a>(
|
||||
self,
|
||||
errors: &'a [Located],
|
||||
bundled: &Catalog,
|
||||
) -> (Self, Vec<Rejection<'a>>) {
|
||||
let (mut catalog, removed) = self.without_rejected(errors);
|
||||
let mut rejected: IndexMap<String, Vec<&'a Located>> = IndexMap::new();
|
||||
for error in removed {
|
||||
if let Some(key) = crate::error::pointer_root(&error.path) {
|
||||
rejected.entry(key).or_default().push(error);
|
||||
}
|
||||
}
|
||||
let rejections = rejected
|
||||
.into_iter()
|
||||
.map(|(key, errors)| {
|
||||
let bundled_entry = bundled.entries.get(&key).cloned();
|
||||
let replaced_by_bundled = bundled_entry.is_some();
|
||||
if let Some(entry) = bundled_entry {
|
||||
catalog
|
||||
.lowercase_keys
|
||||
.insert(key.to_lowercase(), key.clone());
|
||||
catalog.entries.insert(key.clone(), entry);
|
||||
}
|
||||
Rejection {
|
||||
key,
|
||||
errors,
|
||||
replaced_by_bundled,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
(catalog, rejections)
|
||||
}
|
||||
|
||||
/// Keys written as `null`, which a fetched document uses to retire a row it no longer serves (decision 23)
|
||||
pub fn removals(&self) -> impl Iterator<Item = &str> {
|
||||
self.removals.iter().map(String::as_str)
|
||||
}
|
||||
|
||||
pub fn unrecognized(&self) -> impl Iterator<Item = (&str, &str)> {
|
||||
self.entries.iter().flat_map(|(name, entry)| {
|
||||
entry
|
||||
|
|
@ -370,3 +421,55 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Located> {
|
|||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn parse(document: Value) -> Loaded<Catalog> {
|
||||
Catalog::parse(document.to_string().as_bytes()).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_rejected_row_is_replaced_by_bundled_or_left_out() {
|
||||
let bundled =
|
||||
parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}}))
|
||||
.value;
|
||||
let fetched = parse(json!({
|
||||
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1},
|
||||
"acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"},
|
||||
"kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0}
|
||||
}));
|
||||
|
||||
let (catalog, rejections) = fetched
|
||||
.value
|
||||
.with_rejected_replaced(&fetched.errors, &bundled);
|
||||
|
||||
let replaced: Vec<(&str, bool)> = rejections
|
||||
.iter()
|
||||
.map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled))
|
||||
.collect();
|
||||
assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]);
|
||||
assert_eq!(
|
||||
catalog
|
||||
.lookup("gpt-4o")
|
||||
.unwrap()
|
||||
.entry
|
||||
.field("input_cost_per_token"),
|
||||
Some(&json!(1.0))
|
||||
);
|
||||
assert!(catalog.lookup("acme/new").is_none());
|
||||
assert!(catalog.lookup("kept").is_some());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_null_row_is_a_removal_not_an_error() {
|
||||
let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
|
||||
assert!(loaded.errors.is_empty(), "{:?}", loaded.errors);
|
||||
assert_eq!(loaded.value.removals().collect::<Vec<_>>(), ["retired"]);
|
||||
assert_eq!(loaded.value.served().count(), 1);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ use crate::catalog::Catalog;
|
|||
use crate::register::Registrations;
|
||||
use crate::wire::Fields;
|
||||
|
||||
const ALIASES: &str = "aliases";
|
||||
|
||||
/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove`
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Patch {
|
||||
|
|
@ -16,10 +18,6 @@ pub enum Patch {
|
|||
|
||||
/// The four layers served rows are composed from, lowest first (decisions 23 and 25)
|
||||
#[derive(Clone, Debug)]
|
||||
#[expect(
|
||||
dead_code,
|
||||
reason = "02 and 07 add the functions that read these layers"
|
||||
)]
|
||||
pub struct Layers {
|
||||
/// 0, the wheel's embedded map
|
||||
bundled: Arc<Catalog>,
|
||||
|
|
@ -28,5 +26,138 @@ pub struct Layers {
|
|||
/// 2, `register(persist=False)`, cleared by every install and refilled by the router replay
|
||||
transient: IndexMap<String, Fields>,
|
||||
/// 3, `register(persist=True)`, applied over every install
|
||||
#[expect(dead_code, reason = "07 applies registrations over the fetched layer")]
|
||||
persisted: Registrations,
|
||||
}
|
||||
|
||||
impl Patch {
|
||||
pub fn from_value(value: &Value) -> Self {
|
||||
match value {
|
||||
Value::Null => Self::Remove,
|
||||
value => Self::Set(value.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Layers {
|
||||
pub fn new(bundled: Arc<Catalog>) -> Self {
|
||||
Self {
|
||||
bundled,
|
||||
fetched: None,
|
||||
transient: IndexMap::new(),
|
||||
persisted: Registrations::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Install replaces layer 1 and clears layer 2, which the router replay refills
|
||||
pub fn replace_fetched(&mut self, fetched: Arc<Catalog>) {
|
||||
self.fetched = Some(fetched);
|
||||
self.transient.clear();
|
||||
}
|
||||
|
||||
/// What install writes. A fetched row patches the bundled row field by field, a fetched `null` row removes it
|
||||
pub fn compose(&self) -> IndexMap<String, Fields> {
|
||||
let mut rows: IndexMap<String, Fields> = self
|
||||
.bundled
|
||||
.served()
|
||||
.map(|(key, fields)| (key.to_owned(), without_aliases(fields)))
|
||||
.collect();
|
||||
let Some(fetched) = &self.fetched else {
|
||||
return rows;
|
||||
};
|
||||
for key in fetched.removals() {
|
||||
rows.shift_remove(key);
|
||||
}
|
||||
for (key, fields) in fetched.served() {
|
||||
let row = rows.entry(key.to_owned()).or_default();
|
||||
for (field, value) in fields.iter().filter(|(field, _)| *field != ALIASES) {
|
||||
match Patch::from_value(value) {
|
||||
Patch::Set(value) => {
|
||||
row.insert(field.clone(), value);
|
||||
}
|
||||
Patch::Remove => {
|
||||
row.shift_remove(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
/// The canonical key an alias is served under, from the fetched layer first
|
||||
pub fn alias_owner(&self, key: &str) -> Option<&str> {
|
||||
self.fetched
|
||||
.iter()
|
||||
.chain([&self.bundled])
|
||||
.find_map(|catalog| catalog.aliases().get(key))
|
||||
.map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
fn without_aliases(fields: &Fields) -> Fields {
|
||||
fields
|
||||
.iter()
|
||||
.filter(|(field, value)| *field != ALIASES && !value.is_null())
|
||||
.map(|(field, value)| (field.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn catalog(document: Value) -> Arc<Catalog> {
|
||||
Arc::new(
|
||||
Catalog::parse(document.to_string().as_bytes())
|
||||
.unwrap()
|
||||
.value,
|
||||
)
|
||||
}
|
||||
|
||||
fn bundled() -> Arc<Catalog> {
|
||||
catalog(json!({
|
||||
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
|
||||
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0}
|
||||
}))
|
||||
}
|
||||
|
||||
fn composed(fetched: Value) -> IndexMap<String, Fields> {
|
||||
let mut layers = Layers::new(bundled());
|
||||
layers.replace_fetched(catalog(fetched));
|
||||
layers.compose()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_field_the_fetched_row_lacks_is_served_from_bundled() {
|
||||
let rows = composed(
|
||||
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}),
|
||||
);
|
||||
assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0));
|
||||
assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_null_field_removes_it() {
|
||||
let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}));
|
||||
assert!(!rows["gpt-4o"].contains_key("max_tokens"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() {
|
||||
let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}}));
|
||||
let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
|
||||
assert!(kept.contains_key("retired"));
|
||||
assert!(!retired.contains_key("retired"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn aliases_are_served_under_every_name_without_the_aliases_field() {
|
||||
let rows =
|
||||
composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}}));
|
||||
assert_eq!(rows["new-alias"], rows["new"]);
|
||||
assert!(!rows["new"].contains_key("aliases"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -380,6 +380,7 @@ fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
|
|||
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
|
||||
let unrecognized: Vec<_> = catalog.unrecognized().collect();
|
||||
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
|
||||
assert!(catalog.removals().next().is_none());
|
||||
assert!(
|
||||
catalog
|
||||
.fallback_rules()
|
||||
|
|
|
|||
|
|
@ -48,6 +48,9 @@ litellm-secrets-types.workspace = true
|
|||
litellm-llms-types.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
indexmap = "2.14.0"
|
||||
sha1.workspace = true
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
pyo3.workspace = true
|
||||
prost.workspace = true
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ mod execution;
|
|||
mod http;
|
||||
mod logger;
|
||||
mod marshal;
|
||||
mod model_registry;
|
||||
mod preflight;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
|
|
@ -18,7 +19,10 @@ mod tokenizer;
|
|||
mod _native {
|
||||
#[pymodule_export]
|
||||
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
|
||||
use pyo3::{prelude::*, types::PyModule};
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyModule, PyTuple},
|
||||
};
|
||||
|
||||
#[cfg(feature = "panic-test")]
|
||||
#[pymodule_export]
|
||||
|
|
@ -30,6 +34,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable};
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{acompletion, completion};
|
||||
|
|
@ -66,7 +72,18 @@ mod _native {
|
|||
dict.set_item(
|
||||
"_SecretManagerRuntime",
|
||||
py.get_type::<crate::secrets::runtime::NativeSecretManager>(),
|
||||
)
|
||||
)?;
|
||||
let typing = py.import("typing")?;
|
||||
let literal = typing.getattr("Literal")?;
|
||||
let row_error_kinds = PyTuple::new(
|
||||
py,
|
||||
<crate::model_registry::RowErrorKind as strum::VariantArray>::VARIANTS
|
||||
.iter()
|
||||
.map(|kind| literal.get_item(<&'static str>::from(*kind)))
|
||||
.collect::<PyResult<Vec<_>>>()?,
|
||||
)?;
|
||||
let row_error_kind = typing.getattr("Union")?.get_item(row_error_kinds)?;
|
||||
module.add("RowErrorKind", row_error_kind)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -81,6 +98,35 @@ pub(crate) fn native_module(py: Python<'_>) -> Bound<'_, PyModule> {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn singleton_literal_values(py: Python<'_>, alias: &Bound<'_, PyAny>) -> (bool, Vec<String>) {
|
||||
let typing = py.import("typing").unwrap();
|
||||
let literal = typing.getattr("Literal").unwrap();
|
||||
let components = typing.call_method1("get_args", (alias,)).unwrap();
|
||||
let inspected = components
|
||||
.try_iter()
|
||||
.unwrap()
|
||||
.map(|component| {
|
||||
let component = component.unwrap();
|
||||
let origin = typing.call_method1("get_origin", (&component,)).unwrap();
|
||||
let arguments = typing.call_method1("get_args", (&component,)).unwrap();
|
||||
let values = arguments.extract::<Vec<String>>().unwrap();
|
||||
(
|
||||
origin.is(&literal) && values.len() == 1,
|
||||
values.first().cloned(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(
|
||||
inspected
|
||||
.iter()
|
||||
.all(|(is_singleton_literal, _)| *is_singleton_literal),
|
||||
inspected
|
||||
.into_iter()
|
||||
.filter_map(|(_, value)| value)
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn module_registration_preserves_the_public_surface() {
|
||||
Python::initialize();
|
||||
|
|
@ -107,6 +153,12 @@ mod tests {
|
|||
"aresponses",
|
||||
"ResponsesWebSocketConnection",
|
||||
"NativeDiagnosticProcessor",
|
||||
"Catalog",
|
||||
"Fetched",
|
||||
"Installed",
|
||||
"Unavailable",
|
||||
"RowError",
|
||||
"RowErrorKind",
|
||||
"NativeClickHouseSpendConfig",
|
||||
"NativeClickHouseSpendStorage",
|
||||
"trace_encode_error",
|
||||
|
|
@ -132,4 +184,48 @@ mod tests {
|
|||
assert_eq!(public_names, expected);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn row_error_kind_is_a_union_of_singleton_literals() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let module = native_module(py);
|
||||
let typing = py.import("typing").unwrap();
|
||||
let union = typing.getattr("Union").unwrap();
|
||||
let alias = module.getattr("RowErrorKind").unwrap();
|
||||
let origin = typing.call_method1("get_origin", (alias.clone(),)).unwrap();
|
||||
assert!(origin.is(&union));
|
||||
let (all_singleton_literals, actual) = singleton_literal_values(py, &alias);
|
||||
assert!(all_singleton_literals);
|
||||
assert_eq!(
|
||||
actual,
|
||||
Vec::from([
|
||||
"json".to_owned(),
|
||||
"empty".to_owned(),
|
||||
"below_minimum".to_owned(),
|
||||
"shrunk".to_owned(),
|
||||
"not_an_object".to_owned(),
|
||||
"invalid".to_owned(),
|
||||
"unknown_block_key".to_owned(),
|
||||
"missing_provider".to_owned(),
|
||||
"unknown_timezone".to_owned(),
|
||||
"alias_shadows_model".to_owned(),
|
||||
"alias_taken".to_owned(),
|
||||
"duplicate_price".to_owned(),
|
||||
"suffix_inside_block".to_owned(),
|
||||
"tier_range".to_owned(),
|
||||
"tier_reversed".to_owned(),
|
||||
"tier_gap".to_owned(),
|
||||
"mixed_tier_basis".to_owned(),
|
||||
"not_priced_by_results".to_owned(),
|
||||
"bands_from_two_sources".to_owned(),
|
||||
"not_an_off_peak_rate".to_owned(),
|
||||
"no_off_peak_prices".to_owned(),
|
||||
"no_off_peak_hours".to_owned(),
|
||||
"empty_list".to_owned(),
|
||||
"invalid_date".to_owned(),
|
||||
])
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
298
litellm-rust/crates/python-bridge/src/model_registry.rs
Normal file
298
litellm-rust/crates/python-bridge/src/model_registry.rs
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
mod install;
|
||||
mod row_error;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
|
||||
|
||||
use litellm_host_python::{release_gil, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::pybacked::PyBackedBytes;
|
||||
use pyo3::types::{PyDict, PyModule};
|
||||
use serde::Serialize;
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
use install::{Record, write_into};
|
||||
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
|
||||
|
||||
static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
|
||||
let loaded = Catalog::parse(BUNDLED_REGISTRY).expect("the bundled model registry parses");
|
||||
let catalog = Arc::new(loaded.value);
|
||||
Arc::new(Parsed {
|
||||
raw: Arc::clone(&catalog),
|
||||
catalog,
|
||||
errors: loaded.errors.iter().map(row_error).collect(),
|
||||
rejected: Vec::new(),
|
||||
})
|
||||
});
|
||||
|
||||
static BUNDLED_REVISION: LazyLock<String> = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY));
|
||||
|
||||
struct Parsed {
|
||||
/// What install layers over bundled, rows decision 15 rejects replaced or left out
|
||||
catalog: Arc<Catalog>,
|
||||
/// The document as written
|
||||
raw: Arc<Catalog>,
|
||||
errors: Vec<RowError>,
|
||||
rejected: Vec<RowError>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Fetched {
|
||||
parsed: Arc<Parsed>,
|
||||
#[pyo3(get)]
|
||||
revision: String,
|
||||
#[pyo3(get)]
|
||||
etag: Option<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Installed {
|
||||
/// Rows the composed map serves, what main's adopt returns as the fetched map's length
|
||||
#[pyo3(get)]
|
||||
count: usize,
|
||||
/// Keys set because the row differed or was new
|
||||
#[pyo3(get)]
|
||||
written: usize,
|
||||
/// Keys deleted because the map no longer serves them
|
||||
#[pyo3(get)]
|
||||
removed: usize,
|
||||
/// Decision 15, empty for the bundled map
|
||||
#[pyo3(get)]
|
||||
rejected: Vec<RowError>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Unavailable {
|
||||
#[pyo3(get)]
|
||||
reason: String,
|
||||
}
|
||||
|
||||
struct State {
|
||||
layers: Layers,
|
||||
record: Option<Record>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct CatalogHandle {
|
||||
module: Py<PyModule>,
|
||||
attr: String,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Fetched {
|
||||
/// The rows of this one document as install would serve them, without layers
|
||||
fn served(&self, py: Python<'_>) -> PyResult<Py<PyDict>> {
|
||||
let served = release_gil(py, || served_rows(&self.parsed.catalog));
|
||||
let rows = PyDict::new(py);
|
||||
let mut shared: HashMap<String, Py<PyDict>> = HashMap::new();
|
||||
for (key, owner, fields) in served {
|
||||
let row = match shared.get(&owner) {
|
||||
Some(row) => row.clone_ref(py),
|
||||
None => {
|
||||
let row = fields_to_dict(py, &fields)?;
|
||||
shared.insert(owner, row.clone_ref(py));
|
||||
row
|
||||
}
|
||||
};
|
||||
rows.set_item(key, row)?;
|
||||
}
|
||||
Ok(rows.unbind())
|
||||
}
|
||||
|
||||
/// The raw document, `sample_spec` and `fallback_generalizations` included
|
||||
fn document(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, &WrittenDocument(self.parsed.raw.as_ref()))
|
||||
}
|
||||
|
||||
fn fallback_rules(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, &self.parsed.raw.fallback_rules())
|
||||
}
|
||||
|
||||
fn errors(&self) -> Vec<RowError> {
|
||||
self.parsed.errors.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl CatalogHandle {
|
||||
#[staticmethod]
|
||||
fn bound_to(module: Bound<'_, PyModule>, attr: String) -> Self {
|
||||
Self {
|
||||
module: module.unbind(),
|
||||
attr,
|
||||
state: Mutex::new(State {
|
||||
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
|
||||
record: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn bundled(&self) -> Fetched {
|
||||
Fetched {
|
||||
parsed: Arc::clone(&BUNDLED),
|
||||
revision: BUNDLED_REVISION.clone(),
|
||||
etag: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a fetched body and applies main's whole-map checks against the bundled model count
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))]
|
||||
fn parse(
|
||||
py: Python<'_>,
|
||||
body: PyBackedBytes,
|
||||
etag: Option<String>,
|
||||
min_model_count: usize,
|
||||
max_shrink_ratio: f64,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let parsed = release_gil(py, || {
|
||||
parse_remote(&body, min_model_count, max_shrink_ratio)
|
||||
});
|
||||
match parsed {
|
||||
Ok((parsed, revision)) => Ok(Fetched {
|
||||
parsed: Arc::new(parsed),
|
||||
revision,
|
||||
etag,
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
Err(reason) => Ok(Unavailable { reason }
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25)
|
||||
fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult<Installed> {
|
||||
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
|
||||
let dict = bound
|
||||
.cast::<PyDict>()
|
||||
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?;
|
||||
let dict_id = dict.as_ptr() as usize;
|
||||
let (layers, record) = {
|
||||
let mut state = self.lock();
|
||||
state
|
||||
.layers
|
||||
.replace_fetched(Arc::clone(&fetched.parsed.catalog));
|
||||
let record = state
|
||||
.record
|
||||
.clone()
|
||||
.filter(|record| record.dict_id == dict_id);
|
||||
(state.layers.clone(), record)
|
||||
};
|
||||
let composed = release_gil(py, || layers.compose());
|
||||
let outcome = write_into(py, dict, &composed, &layers, record)?;
|
||||
self.lock().record = Some(outcome.record);
|
||||
Ok(Installed {
|
||||
count: composed.len(),
|
||||
written: outcome.written,
|
||||
removed: outcome.removed,
|
||||
rejected: fetched.parsed.rejected.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl CatalogHandle {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_remote(
|
||||
body: &[u8],
|
||||
min_model_count: usize,
|
||||
max_shrink_ratio: f64,
|
||||
) -> Result<(Parsed, String), String> {
|
||||
let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio)
|
||||
.ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?;
|
||||
let loaded = Catalog::parse(body).map_err(|error| error.to_string())?;
|
||||
loaded
|
||||
.value
|
||||
.check_size(BUNDLED.raw.model_count(), limits)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let raw = Arc::new(loaded.value.clone());
|
||||
let errors = loaded.errors.iter().map(row_error).collect();
|
||||
let (catalog, rejections) = loaded
|
||||
.value
|
||||
.with_rejected_replaced(&loaded.errors, &BUNDLED.raw);
|
||||
let rejected = rejections
|
||||
.iter()
|
||||
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
|
||||
.collect();
|
||||
Ok((
|
||||
Parsed {
|
||||
catalog: Arc::new(catalog),
|
||||
raw,
|
||||
errors,
|
||||
rejected,
|
||||
},
|
||||
git_blob_id(body),
|
||||
))
|
||||
}
|
||||
|
||||
fn git_blob_id(body: &[u8]) -> String {
|
||||
let mut hasher = Sha1::new();
|
||||
hasher.update(format!("blob {}\0", body.len()).as_bytes());
|
||||
hasher.update(body);
|
||||
hasher
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> {
|
||||
catalog
|
||||
.served()
|
||||
.map(|(key, fields)| {
|
||||
let owner = catalog
|
||||
.aliases()
|
||||
.get(key)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| key.to_owned());
|
||||
let fields = fields
|
||||
.iter()
|
||||
.filter(|(field, _)| *field != "aliases")
|
||||
.map(|(field, value)| (field.clone(), value.clone()))
|
||||
.collect();
|
||||
(key.to_owned(), owner, fields)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn fields_to_dict(py: Python<'_>, fields: &Fields) -> PyResult<Py<PyDict>> {
|
||||
let row = PyDict::new(py);
|
||||
for (key, value) in fields {
|
||||
row.set_item(key, to_py(py, value)?)?;
|
||||
}
|
||||
Ok(row.unbind())
|
||||
}
|
||||
|
||||
struct WrittenDocument<'a>(&'a Catalog);
|
||||
|
||||
impl Serialize for WrittenDocument<'_> {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.collect_map(self.0.written())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
fn the_revision_is_the_git_blob_id_of_the_body() {
|
||||
assert_eq!(
|
||||
git_blob_id(b"hello\n"),
|
||||
"ce013625030ba8dba906f756967f9e9ca394464a"
|
||||
);
|
||||
}
|
||||
}
|
||||
234
litellm-rust/crates/python-bridge/src/model_registry/install.rs
Normal file
234
litellm-rust/crates/python-bridge/src/model_registry/install.rs
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use litellm_host_python::{from_py, to_py};
|
||||
use litellm_model_catalog::Layers;
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use serde_json::Value;
|
||||
|
||||
/// The field values litellm last wrote into one dict object (decision 25)
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Record {
|
||||
pub(crate) dict_id: usize,
|
||||
rows: HashMap<String, Fields>,
|
||||
}
|
||||
|
||||
pub(crate) struct Outcome {
|
||||
pub(crate) record: Record,
|
||||
pub(crate) written: usize,
|
||||
pub(crate) removed: usize,
|
||||
}
|
||||
|
||||
/// What the dict holds for one field, as litellm compares it
|
||||
#[derive(PartialEq)]
|
||||
enum Held {
|
||||
Absent,
|
||||
Json(Value),
|
||||
/// A value that does not convert to JSON, which only the user can have written
|
||||
Foreign,
|
||||
}
|
||||
|
||||
/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the
|
||||
/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the
|
||||
/// last install is written in full, which is main's result after rebind then adopt
|
||||
pub(crate) fn write_into(
|
||||
py: Python<'_>,
|
||||
dict: &Bound<'_, PyDict>,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
previous: Option<Record>,
|
||||
) -> PyResult<Outcome> {
|
||||
let dict_id = dict.as_ptr() as usize;
|
||||
let Some(previous) = previous else {
|
||||
return write_in_full(py, dict, composed, layers, dict_id);
|
||||
};
|
||||
let mut rows = previous.rows;
|
||||
let mut written = 0;
|
||||
let mut removed = 0;
|
||||
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
|
||||
let keys: Vec<String> = composed
|
||||
.keys()
|
||||
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
|
||||
.cloned()
|
||||
.collect();
|
||||
for key in keys {
|
||||
let target = composed.get(&key);
|
||||
let last = rows.remove(&key);
|
||||
let Some(current) = dict.get_item(&key)? else {
|
||||
match (target, last) {
|
||||
(Some(target), None) => {
|
||||
dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?;
|
||||
rows.insert(key, target.clone());
|
||||
written += 1;
|
||||
}
|
||||
(_, Some(last)) if target.is_some() => {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let Ok(row) = current.cast::<PyDict>() else {
|
||||
if let Some(last) = last {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let last = last.unwrap_or_default();
|
||||
let (owned, changed) = write_owned_fields(py, row, target, &last)?;
|
||||
if target.is_none() && changed && row.is_empty() {
|
||||
dict.del_item(&key)?;
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
written += usize::from(changed);
|
||||
if !owned.is_empty() {
|
||||
rows.insert(key, owned);
|
||||
}
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record { dict_id, rows },
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
fn write_in_full(
|
||||
py: Python<'_>,
|
||||
dict: &Bound<'_, PyDict>,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
dict_id: usize,
|
||||
) -> PyResult<Outcome> {
|
||||
let stale: Vec<Bound<'_, PyAny>> = dict
|
||||
.keys()
|
||||
.iter()
|
||||
.filter(|key| {
|
||||
key.extract::<String>()
|
||||
.map_or(true, |key| !composed.contains_key(&key))
|
||||
})
|
||||
.collect();
|
||||
let removed = stale.len();
|
||||
for key in stale {
|
||||
dict.del_item(key)?;
|
||||
}
|
||||
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
|
||||
let mut written = 0;
|
||||
for (key, target) in composed {
|
||||
let current = dict.get_item(key)?;
|
||||
let row = current.as_ref().and_then(|row| row.cast::<PyDict>().ok());
|
||||
match row {
|
||||
Some(row) => {
|
||||
let owned_by_litellm: Fields = row
|
||||
.iter()
|
||||
.filter_map(|(field, value)| {
|
||||
let field = field.extract::<String>().ok()?;
|
||||
Some((field, from_py::<Value>(&value).ok()?))
|
||||
})
|
||||
.collect();
|
||||
let extra: Vec<String> = row
|
||||
.keys()
|
||||
.iter()
|
||||
.filter_map(|field| field.extract::<String>().ok())
|
||||
.filter(|field| !owned_by_litellm.contains_key(field))
|
||||
.collect();
|
||||
for field in extra {
|
||||
row.del_item(field)?;
|
||||
}
|
||||
let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?;
|
||||
written += usize::from(changed);
|
||||
}
|
||||
None => {
|
||||
dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?;
|
||||
written += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record {
|
||||
dict_id,
|
||||
rows: composed
|
||||
.iter()
|
||||
.map(|(key, fields)| (key.clone(), fields.clone()))
|
||||
.collect(),
|
||||
},
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
|
||||
fn write_owned_fields(
|
||||
py: Python<'_>,
|
||||
row: &Bound<'_, PyDict>,
|
||||
target: Option<&Fields>,
|
||||
last: &Fields,
|
||||
) -> PyResult<(Fields, bool)> {
|
||||
let empty = Fields::new();
|
||||
let target = target.unwrap_or(&empty);
|
||||
let fields: Vec<&String> = target
|
||||
.keys()
|
||||
.chain(last.keys().filter(|field| !target.contains_key(*field)))
|
||||
.collect();
|
||||
let mut owned = Fields::new();
|
||||
let mut changed = false;
|
||||
for field in fields {
|
||||
let held = held(row, field)?;
|
||||
let litellm_owns = match (&held, last.get(field)) {
|
||||
(Held::Absent, None) => true,
|
||||
(Held::Json(value), Some(last)) => value == last,
|
||||
_ => false,
|
||||
};
|
||||
if !litellm_owns {
|
||||
if let Some(last) = last.get(field) {
|
||||
owned.insert(field.clone(), last.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match target.get(field) {
|
||||
Some(value) => {
|
||||
if held != Held::Json(value.clone()) {
|
||||
row.set_item(field, to_py(py, value)?)?;
|
||||
changed = true;
|
||||
}
|
||||
owned.insert(field.clone(), value.clone());
|
||||
}
|
||||
None if held != Held::Absent => {
|
||||
row.del_item(field)?;
|
||||
changed = true;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
Ok((owned, changed))
|
||||
}
|
||||
|
||||
fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult<Held> {
|
||||
Ok(match row.get_item(field)? {
|
||||
None => Held::Absent,
|
||||
Some(value) => from_py::<Value>(&value).map_or(Held::Foreign, Held::Json),
|
||||
})
|
||||
}
|
||||
|
||||
/// A new row, the same object for every key of one alias group, as main's alias expansion gives them
|
||||
fn shared_row(
|
||||
py: Python<'_>,
|
||||
fresh: &mut HashMap<String, Py<PyDict>>,
|
||||
layers: &Layers,
|
||||
key: &str,
|
||||
fields: &Fields,
|
||||
) -> PyResult<Py<PyDict>> {
|
||||
let owner = layers.alias_owner(key).unwrap_or(key).to_owned();
|
||||
if let Some(row) = fresh.get(&owner) {
|
||||
return Ok(row.clone_ref(py));
|
||||
}
|
||||
let row = PyDict::new(py);
|
||||
for (field, value) in fields {
|
||||
row.set_item(field, to_py(py, value)?)?;
|
||||
}
|
||||
let row = row.unbind();
|
||||
fresh.insert(owner, row.clone_ref(py));
|
||||
Ok(row)
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
use litellm_model_catalog::{Error, Located};
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
|
||||
pub(crate) enum RowErrorKind {
|
||||
#[strum(serialize = "json")]
|
||||
Json,
|
||||
#[strum(serialize = "empty")]
|
||||
Empty,
|
||||
#[strum(serialize = "below_minimum")]
|
||||
BelowMinimum,
|
||||
#[strum(serialize = "shrunk")]
|
||||
Shrunk,
|
||||
#[strum(serialize = "not_an_object")]
|
||||
NotAnObject,
|
||||
#[strum(serialize = "invalid")]
|
||||
Invalid,
|
||||
#[strum(serialize = "unknown_block_key")]
|
||||
UnknownBlockKey,
|
||||
#[strum(serialize = "missing_provider")]
|
||||
MissingProvider,
|
||||
#[strum(serialize = "unknown_timezone")]
|
||||
UnknownTimezone,
|
||||
#[strum(serialize = "alias_shadows_model")]
|
||||
AliasShadowsModel,
|
||||
#[strum(serialize = "alias_taken")]
|
||||
AliasTaken,
|
||||
#[strum(serialize = "duplicate_price")]
|
||||
DuplicatePrice,
|
||||
#[strum(serialize = "suffix_inside_block")]
|
||||
SuffixInsideBlock,
|
||||
#[strum(serialize = "tier_range")]
|
||||
TierRange,
|
||||
#[strum(serialize = "tier_reversed")]
|
||||
TierReversed,
|
||||
#[strum(serialize = "tier_gap")]
|
||||
TierGap,
|
||||
#[strum(serialize = "mixed_tier_basis")]
|
||||
MixedTierBasis,
|
||||
#[strum(serialize = "not_priced_by_results")]
|
||||
NotPricedByResults,
|
||||
#[strum(serialize = "bands_from_two_sources")]
|
||||
BandsFromTwoSources,
|
||||
#[strum(serialize = "not_an_off_peak_rate")]
|
||||
NotAnOffPeakRate,
|
||||
#[strum(serialize = "no_off_peak_prices")]
|
||||
NoOffPeakPrices,
|
||||
#[strum(serialize = "no_off_peak_hours")]
|
||||
NoOffPeakHours,
|
||||
#[strum(serialize = "empty_list")]
|
||||
EmptyList,
|
||||
#[strum(serialize = "invalid_date")]
|
||||
InvalidDate,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RowError {
|
||||
#[pyo3(get)]
|
||||
pub(crate) key: String,
|
||||
#[pyo3(get)]
|
||||
pub(crate) field: Option<String>,
|
||||
#[pyo3(get)]
|
||||
pub(crate) kind: String,
|
||||
#[pyo3(get)]
|
||||
pub(crate) rejects: bool,
|
||||
#[pyo3(get)]
|
||||
pub(crate) message: String,
|
||||
}
|
||||
|
||||
pub(crate) fn row_error(error: &Located) -> RowError {
|
||||
let mut path = error.path.split('/').skip(1);
|
||||
let key = unescape_pointer_segment(path.next().unwrap_or_default());
|
||||
let field = path.map(unescape_pointer_segment).collect::<Vec<_>>();
|
||||
let field = (!field.is_empty()).then(|| field.join("/"));
|
||||
RowError {
|
||||
key,
|
||||
field,
|
||||
kind: <&'static str>::from(error_kind(&error.error)).to_owned(),
|
||||
rejects: litellm_model_catalog::rejects_row(&error.error),
|
||||
message: error.error.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn unescape_pointer_segment(segment: &str) -> String {
|
||||
segment.replace("~1", "/").replace("~0", "~")
|
||||
}
|
||||
|
||||
fn error_kind(error: &Error) -> RowErrorKind {
|
||||
match error {
|
||||
Error::Json(_) => RowErrorKind::Json,
|
||||
Error::Empty => RowErrorKind::Empty,
|
||||
Error::BelowMinimum { .. } => RowErrorKind::BelowMinimum,
|
||||
Error::Shrunk { .. } => RowErrorKind::Shrunk,
|
||||
Error::NotAnObject => RowErrorKind::NotAnObject,
|
||||
Error::Invalid(_) => RowErrorKind::Invalid,
|
||||
Error::UnknownBlockKey => RowErrorKind::UnknownBlockKey,
|
||||
Error::MissingProvider => RowErrorKind::MissingProvider,
|
||||
Error::UnknownTimezone { .. } => RowErrorKind::UnknownTimezone,
|
||||
Error::AliasShadowsModel { .. } => RowErrorKind::AliasShadowsModel,
|
||||
Error::AliasTaken { .. } => RowErrorKind::AliasTaken,
|
||||
Error::DuplicatePrice { .. } => RowErrorKind::DuplicatePrice,
|
||||
Error::SuffixInsideBlock { .. } => RowErrorKind::SuffixInsideBlock,
|
||||
Error::TierRange => RowErrorKind::TierRange,
|
||||
Error::TierReversed => RowErrorKind::TierReversed,
|
||||
Error::TierGap { .. } => RowErrorKind::TierGap,
|
||||
Error::MixedTierBasis => RowErrorKind::MixedTierBasis,
|
||||
Error::NotPricedByResults => RowErrorKind::NotPricedByResults,
|
||||
Error::BandsFromTwoSources => RowErrorKind::BandsFromTwoSources,
|
||||
Error::NotAnOffPeakRate => RowErrorKind::NotAnOffPeakRate,
|
||||
Error::NoOffPeakPrices => RowErrorKind::NoOffPeakPrices,
|
||||
Error::NoOffPeakHours => RowErrorKind::NoOffPeakHours,
|
||||
Error::EmptyList => RowErrorKind::EmptyList,
|
||||
Error::InvalidDate => RowErrorKind::InvalidDate,
|
||||
}
|
||||
}
|
||||
|
|
@ -548,9 +548,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None
|
|||
#### PII MASKING ####
|
||||
output_parse_pii: bool = False
|
||||
#############################################
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
get_model_cost_map as get_model_cost_map,
|
||||
install_initial_model_cost_map,
|
||||
mark_litellm_import_complete,
|
||||
)
|
||||
|
||||
model_cost = get_model_cost_map(url=model_cost_map_url)
|
||||
model_cost: dict[str, dict[str, object]] = {}
|
||||
install_initial_model_cost_map(url=model_cost_map_url)
|
||||
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
||||
apply_search_free_tier: bool = False
|
||||
cost_margin_config: Dict[
|
||||
|
|
|
|||
10
litellm/_catalog/__init__.py
Normal file
10
litellm/_catalog/__init__.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
"""Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release"""
|
||||
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable
|
||||
|
||||
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
|
||||
|
||||
__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"]
|
||||
|
|
@ -2794,7 +2794,7 @@ def default_image_cost_calculator(
|
|||
price * units
|
||||
for price_table in price_tables
|
||||
for cost_key, units in unit_counts
|
||||
if (price := price_table.get(cost_key)) is not None
|
||||
if isinstance(price := price_table.get(cost_key), (int, float))
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
|
|
@ -19,19 +17,19 @@ import threading
|
|||
import time
|
||||
import urllib.parse
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from importlib.resources import files
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable
|
||||
from litellm.constants import (
|
||||
MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
|
|
@ -43,8 +41,6 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
|||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
|
||||
|
||||
|
||||
|
|
@ -52,49 +48,8 @@ def _is_cli_process() -> bool:
|
|||
return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES
|
||||
|
||||
|
||||
# Reserved top-level keys that are not model entries. They must be excluded
|
||||
# from the model-count integrity check so a real upstream shrink can't be masked.
|
||||
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY})
|
||||
|
||||
|
||||
def _count_model_entries(model_cost: dict) -> int:
|
||||
"""Count actual model entries, excluding reserved meta keys."""
|
||||
return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS)
|
||||
|
||||
|
||||
def git_blob_id(body: bytes) -> str:
|
||||
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
class GetModelCostMap:
|
||||
"""
|
||||
Handles fetching, validating, and loading the model cost map.
|
||||
|
||||
Only the backup model *count* is cached (a single int). The full
|
||||
backup dict is never held in memory — it is only parsed when it
|
||||
needs to be *returned* as a fallback.
|
||||
"""
|
||||
|
||||
_backup_model_count: int = -1 # -1 = not yet loaded
|
||||
|
||||
@staticmethod
|
||||
def read_local_model_cost_map_bytes() -> bytes:
|
||||
return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes()
|
||||
|
||||
@staticmethod
|
||||
def read_local_model_cost_map_text() -> str:
|
||||
return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded":
|
||||
body: Final = GetModelCostMap.read_local_model_cost_map_bytes()
|
||||
content: Final = _BUNDLED_CATALOG_ADAPTER.validate_python(json.loads(body))
|
||||
return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body))
|
||||
|
||||
@staticmethod
|
||||
def load_local_model_cost_map() -> dict:
|
||||
"""Load the local backup model cost map bundled with the package."""
|
||||
return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map
|
||||
"""The catalog as last loaded, for readers that want it without later registrations"""
|
||||
|
||||
_loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({})
|
||||
_loaded_catalog_fields: frozenset[str] = frozenset()
|
||||
|
|
@ -115,102 +70,6 @@ class GetModelCostMap:
|
|||
cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()})
|
||||
cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values()))
|
||||
|
||||
@classmethod
|
||||
def get_backup_model_count(cls) -> int:
|
||||
"""Return the number of models in the local backup (cached int)."""
|
||||
if cls._backup_model_count < 0:
|
||||
backup: Final = cls.load_local_model_cost_map()
|
||||
cls._backup_model_count = _count_model_entries(backup)
|
||||
return cls._backup_model_count
|
||||
|
||||
_get_backup_model_count = get_backup_model_count
|
||||
|
||||
@staticmethod
|
||||
def _check_is_valid_dict(fetched_map: dict) -> bool:
|
||||
"""Check 1: fetched map is a non-empty dict."""
|
||||
if not isinstance(fetched_map, dict):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map is not a dict (type=%s). Falling back to local backup.",
|
||||
type(fetched_map).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
if len(fetched_map) == 0:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map is empty. Falling back to local backup.",
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _check_model_count_not_reduced(
|
||||
cls,
|
||||
fetched_map: dict,
|
||||
backup_model_count: int,
|
||||
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
) -> bool:
|
||||
"""Check 2: model count has not reduced significantly vs backup."""
|
||||
fetched_count: Final = _count_model_entries(fetched_map)
|
||||
|
||||
if fetched_count < min_model_count:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
|
||||
"This may indicate a corrupted upstream file. "
|
||||
"Falling back to local backup.",
|
||||
fetched_count,
|
||||
min_model_count,
|
||||
)
|
||||
return False
|
||||
|
||||
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map shrank significantly "
|
||||
"(fetched=%d, backup=%d, threshold=%.0f%%). "
|
||||
"This may indicate a corrupted upstream file. "
|
||||
"Falling back to local backup.",
|
||||
fetched_count,
|
||||
backup_model_count,
|
||||
max_shrink_ratio * 100,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_model_cost_map(
|
||||
cls,
|
||||
fetched_map: dict,
|
||||
backup_model_count: int,
|
||||
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
) -> bool:
|
||||
"""
|
||||
Validate the integrity of a fetched model cost map.
|
||||
|
||||
Runs each check in order and returns False on the first failure.
|
||||
|
||||
Checks:
|
||||
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
|
||||
2. ``_check_model_count_not_reduced`` -- model count meets minimum
|
||||
and has not shrunk >``max_shrink_ratio`` vs backup.
|
||||
|
||||
Returns True if all checks pass, False otherwise.
|
||||
"""
|
||||
if not cls._check_is_valid_dict(fetched_map):
|
||||
return False
|
||||
|
||||
if not cls._check_model_count_not_reduced(
|
||||
fetched_map=fetched_map,
|
||||
backup_model_count=backup_model_count,
|
||||
min_model_count=min_model_count,
|
||||
max_shrink_ratio=max_shrink_ratio,
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
|
||||
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
|
||||
|
|
@ -224,9 +83,15 @@ def mark_litellm_import_complete() -> None:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelCostMapReloaded:
|
||||
model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict
|
||||
revision: str | None = None
|
||||
etag: str | None = None
|
||||
fetched: Fetched
|
||||
|
||||
@property
|
||||
def revision(self) -> str:
|
||||
return self.fetched.revision
|
||||
|
||||
@property
|
||||
def etag(self) -> str | None:
|
||||
return self.fetched.etag
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -314,16 +179,15 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
|
|||
|
||||
|
||||
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
|
||||
try:
|
||||
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
|
||||
except ValueError as e:
|
||||
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
|
||||
if not isinstance(parsed, dict):
|
||||
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
|
||||
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
|
||||
dict[str, object], parsed
|
||||
parsed: Final = CATALOG.parse(
|
||||
content,
|
||||
etag=etag,
|
||||
min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
)
|
||||
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
|
||||
if isinstance(parsed, Unavailable):
|
||||
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}")
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
|
||||
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
|
||||
|
|
@ -465,7 +329,7 @@ async def refetch_model_cost_map(
|
|||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision())
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled()))
|
||||
|
||||
result: Final = await _fetch_remote_model_cost_map_with_retry(
|
||||
url=url,
|
||||
|
|
@ -481,17 +345,12 @@ async def refetch_model_cost_map(
|
|||
result.reason,
|
||||
)
|
||||
return result
|
||||
if not GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=result.model_cost_map,
|
||||
backup_model_count=GetModelCostMap.get_backup_model_count(),
|
||||
):
|
||||
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation")
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_loaded_model_cost_map(result)
|
||||
return _note_loaded(result)
|
||||
|
||||
|
||||
class ModelCostMapSourceInfo:
|
||||
|
|
@ -571,103 +430,46 @@ class AliasedCostEntry(dict[str, object]):
|
|||
self.registered_fields: frozenset[str] = registered_fields
|
||||
|
||||
|
||||
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
|
||||
for owner in dict.fromkeys(alias_owners.values())
|
||||
}
|
||||
def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
|
||||
_cost_map_source_info.source_revision = loaded.fetched.revision
|
||||
_cost_map_source_info.etag = loaded.fetched.etag
|
||||
set_fallback_generalizations(loaded.fetched.fallback_rules())
|
||||
GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module
|
||||
loaded.fetched.served()
|
||||
)
|
||||
return loaded
|
||||
|
||||
|
||||
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
|
||||
"""
|
||||
Expand ``aliases`` lists in model cost entries into top-level entries.
|
||||
class _InstallGeneration:
|
||||
"""Bumped by every install, so a registration racing a reload into the same dict can rebuild its entry"""
|
||||
|
||||
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
|
||||
canonical entry, which carries every name it is served under. The
|
||||
``aliases`` key is removed from the entry so downstream code never sees it.
|
||||
|
||||
If an alias collides with an existing canonical entry the alias is
|
||||
skipped and a warning is logged.
|
||||
"""
|
||||
alias_owners: Final[dict[str, str]] = {}
|
||||
keys_with_aliases: Final[list[str]] = []
|
||||
|
||||
for model_name, model_info in model_cost.items():
|
||||
aliases: object = model_info.get("aliases")
|
||||
if aliases is None:
|
||||
continue
|
||||
keys_with_aliases.append(model_name)
|
||||
if not isinstance(aliases, list):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM model alias field for '%s' is not a list (got %s) — skipping.",
|
||||
model_name,
|
||||
type(aliases).__name__,
|
||||
)
|
||||
continue
|
||||
if not aliases:
|
||||
continue
|
||||
for alias in aliases:
|
||||
if alias in model_cost:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM model alias conflict: alias '%s' (from '%s') "
|
||||
"already exists as a canonical entry — skipping.",
|
||||
alias,
|
||||
model_name,
|
||||
)
|
||||
continue
|
||||
if alias in alias_owners:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM model alias conflict: alias '%s' (from '%s') "
|
||||
"was already claimed by another entry — skipping.",
|
||||
alias,
|
||||
model_name,
|
||||
)
|
||||
continue
|
||||
alias_owners[alias] = model_name
|
||||
|
||||
# Remove the ``aliases`` key from entries so it doesn't pollute model info
|
||||
for key in keys_with_aliases:
|
||||
model_cost[key].pop("aliases", None)
|
||||
|
||||
for owner, names in _alias_groups(alias_owners).items():
|
||||
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
|
||||
return model_cost
|
||||
value: int = 0
|
||||
|
||||
|
||||
def _finalize_model_cost_map(model_cost: dict) -> dict:
|
||||
"""Extract fallback generalizations out of the raw map, then expand aliases.
|
||||
|
||||
The ``fallback_generalizations`` block is installed into the generalizations
|
||||
module and removed from the map so it is never treated as a model entry.
|
||||
"""
|
||||
raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None)
|
||||
rules: Final = raw.get("rules") if isinstance(raw, dict) else None
|
||||
set_fallback_generalizations(rules)
|
||||
return _expand_model_aliases(model_cost)
|
||||
def model_cost_install_generation() -> int:
|
||||
return _InstallGeneration.value
|
||||
|
||||
|
||||
def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
|
||||
_cost_map_source_info.source_revision = loaded.revision
|
||||
_cost_map_source_info.etag = loaded.etag
|
||||
finalized: Final = _finalize_model_cost_map(loaded.model_cost_map)
|
||||
GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module
|
||||
return replace(loaded, model_cost_map=finalized)
|
||||
def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
|
||||
installed: Final = catalog.install(fetched)
|
||||
_InstallGeneration.value += 1
|
||||
if installed.rejected:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: model cost map rows failed validation and are served from the bundled map or left out: %s",
|
||||
"; ".join(f"{error.key}: {error.message}" for error in installed.rejected),
|
||||
)
|
||||
return installed
|
||||
|
||||
|
||||
def adopt_model_cost_map(
|
||||
new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract
|
||||
) -> int:
|
||||
def adopt_model_cost_map(fetched: Fetched) -> int:
|
||||
import litellm
|
||||
from litellm import utils
|
||||
|
||||
litellm.model_cost = new_model_cost_map
|
||||
installed: Final = _install(fetched)
|
||||
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
|
||||
litellm.add_known_models(model_cost_map=new_model_cost_map)
|
||||
fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
utils.reapply_runtime_model_cost_registrations()
|
||||
return fetched_model_count
|
||||
return installed.count
|
||||
|
||||
|
||||
def _retry_remote_fetch_in_background(
|
||||
|
|
@ -700,45 +502,27 @@ def _retry_remote_fetch_in_background(
|
|||
)
|
||||
return
|
||||
_litellm_import_complete.wait()
|
||||
if not GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=result.model_cost_map,
|
||||
backup_model_count=GetModelCostMap.get_backup_model_count(),
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",
|
||||
url,
|
||||
)
|
||||
return
|
||||
finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map
|
||||
_note_loaded(result)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
adopt_model_cost_map(finalized)
|
||||
adopt_model_cost_map(result.fetched)
|
||||
except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays
|
||||
verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e)
|
||||
|
||||
|
||||
def get_model_cost_map(
|
||||
def _pick_model_cost_map(
|
||||
url: str,
|
||||
timeout: int = 5,
|
||||
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
client: "_SyncGetClient | None" = None,
|
||||
) -> dict:
|
||||
timeout: int,
|
||||
max_attempts: int,
|
||||
sleep: Callable[[float], None],
|
||||
rng: random.Random | None,
|
||||
client: "_SyncGetClient | None",
|
||||
) -> Fetched:
|
||||
"""
|
||||
Public entry point — returns the model cost map dict.
|
||||
|
||||
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set or this is a ``lite`` /
|
||||
``litellm-proxy`` CLI process, uses the local backup only.
|
||||
2. Otherwise fetches from ``url``, retrying transient HTTP errors
|
||||
(429/5xx/transport) with Retry-After-aware backoff in a background
|
||||
thread, validates integrity, and falls back to the local backup on any
|
||||
failure.
|
||||
|
||||
Only the backup model count is cached (a single int) for validation.
|
||||
The full backup dict is only parsed when it must be *returned* as a
|
||||
fallback — it is never held in memory long-term.
|
||||
The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a
|
||||
``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient
|
||||
failure, and the bundled map on any failure
|
||||
"""
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
|
|
@ -748,7 +532,7 @@ def get_model_cost_map(
|
|||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
|
||||
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
|
|
@ -779,22 +563,39 @@ def get_model_cost_map(
|
|||
)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}"
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
content: Final = outcome.model_cost_map
|
||||
|
||||
# Validate using cached count (cheap int comparison, no file I/O)
|
||||
if not GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=content,
|
||||
backup_model_count=GetModelCostMap.get_backup_model_count(),
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",
|
||||
url,
|
||||
)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.fallback_reason = "Remote data failed integrity validation"
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
|
||||
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_loaded_model_cost_map(outcome).model_cost_map
|
||||
return _note_loaded(outcome).fetched
|
||||
|
||||
|
||||
def install_initial_model_cost_map(url: str) -> None:
|
||||
"""Fills ``litellm.model_cost`` at import with the map main's rules pick"""
|
||||
_install(
|
||||
_pick_model_cost_map(
|
||||
url=url,
|
||||
timeout=5,
|
||||
max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep=time.sleep,
|
||||
rng=None,
|
||||
client=None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_model_cost_map(
|
||||
url: str,
|
||||
timeout: int = 5,
|
||||
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
client: "_SyncGetClient | None" = None,
|
||||
) -> dict[str, dict[str, JsonValue]]:
|
||||
"""
|
||||
Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and
|
||||
validated by the Rust catalog, and the bundled map is returned on any failure
|
||||
"""
|
||||
return _pick_model_cost_map(
|
||||
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client
|
||||
).served()
|
||||
|
|
|
|||
|
|
@ -805,7 +805,8 @@ class StandardBuiltInToolCostTracking:
|
|||
model_key = model_info.get("key") if isinstance(model_info, dict) else getattr(model_info, "key", None)
|
||||
|
||||
if model_key and model_key in litellm.model_cost:
|
||||
return litellm.model_cost[model_key].get("code_interpreter_cost_per_session")
|
||||
session_cost: Final = litellm.model_cost[model_key].get("code_interpreter_cost_per_session")
|
||||
return float(session_cost) if isinstance(session_cost, (int, float)) else None
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ def _input_cost_per_pixel(resolved: ModelInfo) -> float:
|
|||
shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None
|
||||
if shared_entry is None:
|
||||
return 0.0
|
||||
return shared_entry.get("input_cost_per_pixel") or 0.0
|
||||
shared_price: Final = shared_entry.get("input_cost_per_pixel")
|
||||
return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0
|
||||
|
||||
|
||||
def cost_calculator(
|
||||
|
|
|
|||
|
|
@ -200,13 +200,6 @@ def get_cached_model_info():
|
|||
return _get_model_info
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _get_local_model_cost_map() -> dict:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
|
||||
def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None:
|
||||
"""
|
||||
Remove and return Anthropic's nested ``output_config.format`` field.
|
||||
|
|
@ -1532,12 +1525,6 @@ def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) ->
|
|||
flag = model_info.get(key)
|
||||
if isinstance(flag, bool):
|
||||
return flag
|
||||
|
||||
model_cost_key = model_info.get("key")
|
||||
if isinstance(model_cost_key, str):
|
||||
local_flag = _get_local_model_cost_map().get(model_cost_key, {}).get(key)
|
||||
if isinstance(local_flag, bool):
|
||||
return local_flag
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -1579,16 +1566,7 @@ def _get_bedrock_output_config_effort_ceiling(
|
|||
except Exception:
|
||||
return None
|
||||
|
||||
ceiling = model_info.get("bedrock_output_config_effort_ceiling")
|
||||
if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER:
|
||||
return ceiling
|
||||
|
||||
model_cost_key: Final = model_info.get("key")
|
||||
if not isinstance(model_cost_key, str):
|
||||
return None
|
||||
|
||||
local_model_info: Final = _get_local_model_cost_map().get(model_cost_key, {})
|
||||
ceiling = local_model_info.get("bedrock_output_config_effort_ceiling")
|
||||
ceiling: Final = model_info.get("bedrock_output_config_effort_ceiling")
|
||||
if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER:
|
||||
return ceiling
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
safe_convert_created_field,
|
||||
)
|
||||
|
|
@ -209,7 +208,9 @@ def _bundled_openai_reasoning_models() -> frozenset[str]:
|
|||
map it fetches), and a lagging entry must never strip `reasoning` from a model this
|
||||
release knows accepts it.
|
||||
"""
|
||||
bundled: Final = _BUNDLED_COST_MAP.validate_json(GetModelCostMap.read_local_model_cost_map_text())
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
bundled: Final = _BUNDLED_COST_MAP.validate_python(CATALOG.bundled().document())
|
||||
return frozenset(
|
||||
name
|
||||
for name, entry in bundled.items()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from collections.abc import Callable, Iterable, Mapping, Sequence
|
|||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from itertools import accumulate, chain, repeat, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
|
||||
|
|
@ -77,19 +76,6 @@ def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyri
|
|||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
return MappingProxyType(
|
||||
{
|
||||
model_key: lyria_model_info
|
||||
for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items()
|
||||
if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _vertex_ai_lyria_model_key(model: str) -> str:
|
||||
return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
|
||||
|
||||
|
|
@ -100,18 +86,12 @@ def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None)
|
|||
|
||||
def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None:
|
||||
model_key: Final = _vertex_ai_lyria_model_key(model)
|
||||
runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
|
||||
return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key)
|
||||
return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
|
||||
|
||||
|
||||
def get_vertex_ai_lyria_generation_cost(model: str) -> float | None:
|
||||
model_key: Final = _vertex_ai_lyria_model_key(model)
|
||||
runtime_cost: Final = _vertex_ai_lyria_generation_cost(
|
||||
_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
|
||||
)
|
||||
if runtime_cost is not None:
|
||||
return runtime_cost
|
||||
return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key))
|
||||
return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)))
|
||||
|
||||
|
||||
class VertexAIError(BaseLLMException):
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from collections.abc import Mapping
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
|
||||
|
|
@ -57,6 +59,8 @@ else:
|
|||
GoogleCredentialsObject = Any
|
||||
|
||||
|
||||
_SUPPORTED_REGIONS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
class _VertexCredentialsObject(Protocol):
|
||||
"""Structural view of the google-auth credentials handle that this class caches and refreshes."""
|
||||
|
||||
|
|
@ -114,8 +118,12 @@ class VertexBase:
|
|||
# Try to get supported_regions directly from model_cost
|
||||
# Check both with and without vertex_ai/ prefix
|
||||
model_key: Final = f"vertex_ai/{model}" if not model.startswith("vertex_ai/") else model
|
||||
model_info: Final = litellm.model_cost.get(model_key, {})
|
||||
supported_regions: Final = model_info.get("supported_regions")
|
||||
try:
|
||||
supported_regions: Final = _SUPPORTED_REGIONS.validate_python(
|
||||
litellm.model_cost.get(model_key, {}).get("supported_regions") or ()
|
||||
)
|
||||
except ValidationError:
|
||||
return vertex_region or "us-central1"
|
||||
|
||||
if supported_regions and len(supported_regions) > 0:
|
||||
# If user didn't specify region, use the first supported region
|
||||
|
|
|
|||
|
|
@ -711,7 +711,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -
|
|||
"""
|
||||
for deployment in deployments:
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
if not isinstance(model_id, str):
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
|
||||
|
|
@ -729,7 +729,7 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b
|
|||
"""
|
||||
for deployment in deployments:
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
if not isinstance(model_id, str):
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(
|
||||
|
|
|
|||
|
|
@ -211,6 +211,7 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span as _Span
|
||||
from prisma import models as prisma_models
|
||||
|
||||
from litellm._catalog import Fetched
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager
|
||||
|
||||
|
|
@ -5385,10 +5386,10 @@ def resolve_classifier_plugin(
|
|||
return resolved
|
||||
|
||||
|
||||
def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
|
||||
def _swap_in_model_cost_map(fetched: "Fetched") -> int:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
return adopt_model_cost_map(new_model_cost_map)
|
||||
return adopt_model_cost_map(fetched)
|
||||
|
||||
|
||||
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
|
||||
|
|
@ -9104,7 +9105,7 @@ class ProxyConfig:
|
|||
)
|
||||
return
|
||||
|
||||
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
|
||||
models_count = _swap_in_model_cost_map(reload_result.fetched)
|
||||
self.model_cost_map_loaded_at = current_time
|
||||
await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time)
|
||||
# Adopted last, so neither a failed fetch nor a failed status write is recorded
|
||||
|
|
@ -21011,7 +21012,7 @@ async def reload_model_cost_map(
|
|||
detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.",
|
||||
)
|
||||
|
||||
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
|
||||
models_count = _swap_in_model_cost_map(reload_result.fetched)
|
||||
current_time = utc_now()
|
||||
proxy_config.model_cost_map_loaded_at = current_time
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
|
|
|
|||
|
|
@ -12284,9 +12284,9 @@ class Router:
|
|||
# tell a resolvable name apart; require usable limits/costs.
|
||||
_azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}"
|
||||
_fallback_entry = litellm.model_cost.get(_azure_fallback_key)
|
||||
_fallback_resolves = _fallback_entry is not None and (
|
||||
(_fallback_entry.get("max_input_tokens") or 0) > 0
|
||||
or (_fallback_entry.get("input_cost_per_token") or 0) > 0
|
||||
_fallback_resolves = _fallback_entry is not None and any(
|
||||
isinstance(limit, (int, float)) and limit > 0
|
||||
for limit in (_fallback_entry.get("max_input_tokens"), _fallback_entry.get("input_cost_per_token"))
|
||||
)
|
||||
if _fallback_resolves:
|
||||
verbose_router_logger.debug(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from asyncio import Future
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Never, final
|
||||
from types import ModuleType
|
||||
from typing import Literal, Never, final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -15,6 +16,72 @@ class ProcessReservedForForking(RuntimeError): ...
|
|||
|
||||
def trace_encode_error(message: str) -> bytes: ...
|
||||
|
||||
RowErrorKind = Literal[
|
||||
"json",
|
||||
"empty",
|
||||
"below_minimum",
|
||||
"shrunk",
|
||||
"not_an_object",
|
||||
"invalid",
|
||||
"unknown_block_key",
|
||||
"missing_provider",
|
||||
"unknown_timezone",
|
||||
"alias_shadows_model",
|
||||
"alias_taken",
|
||||
"duplicate_price",
|
||||
"suffix_inside_block",
|
||||
"tier_range",
|
||||
"tier_reversed",
|
||||
"tier_gap",
|
||||
"mixed_tier_basis",
|
||||
"not_priced_by_results",
|
||||
"bands_from_two_sources",
|
||||
"not_an_off_peak_rate",
|
||||
"no_off_peak_prices",
|
||||
"no_off_peak_hours",
|
||||
"empty_list",
|
||||
"invalid_date",
|
||||
]
|
||||
|
||||
@final
|
||||
class RowError:
|
||||
key: str
|
||||
field: str | None
|
||||
kind: RowErrorKind
|
||||
rejects: bool
|
||||
message: str
|
||||
|
||||
@final
|
||||
class Fetched:
|
||||
revision: str
|
||||
etag: str | None
|
||||
def served(self) -> dict[str, dict[str, JsonValue]]: ...
|
||||
def document(self) -> dict[str, JsonValue]: ...
|
||||
def fallback_rules(self) -> list[JsonValue] | None: ...
|
||||
def errors(self) -> list[RowError]: ...
|
||||
|
||||
@final
|
||||
class Installed:
|
||||
count: int
|
||||
written: int
|
||||
removed: int
|
||||
rejected: list[RowError]
|
||||
|
||||
@final
|
||||
class Unavailable:
|
||||
reason: str
|
||||
|
||||
@final
|
||||
class Catalog:
|
||||
@staticmethod
|
||||
def bound_to(module: ModuleType, attr: str) -> Catalog: ...
|
||||
def bundled(self) -> Fetched: ...
|
||||
@staticmethod
|
||||
def parse(
|
||||
body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float
|
||||
) -> Fetched | Unavailable: ...
|
||||
def install(self, fetched: Fetched) -> Installed: ...
|
||||
|
||||
@final
|
||||
class NativeClickHouseSpendConfig:
|
||||
def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ...
|
||||
|
|
|
|||
|
|
@ -3830,10 +3830,19 @@ def _registration_attempt(
|
|||
warning_display_name: str | None,
|
||||
final_attempt: bool,
|
||||
) -> _RegistrationWrite | None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import model_cost_install_generation
|
||||
|
||||
source: Final = _builtin_cost_map()
|
||||
generation: Final = model_cost_install_generation()
|
||||
write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name)
|
||||
if not final_attempt and model_cost_install_generation() != generation:
|
||||
return None
|
||||
_replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry, value)
|
||||
return write if final_attempt or litellm.model_cost is source else None
|
||||
return (
|
||||
write
|
||||
if final_attempt or (litellm.model_cost is source and model_cost_install_generation() == generation)
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _register_on_current_map(
|
||||
|
|
@ -6041,6 +6050,10 @@ def get_utc_datetime() -> datetime.datetime:
|
|||
return datetime.datetime.now(datetime.timezone.utc)
|
||||
|
||||
|
||||
def _as_max_tokens(value: object) -> int | None:
|
||||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def get_max_tokens(model: str) -> int | None:
|
||||
"""
|
||||
Get the maximum number of output tokens allowed for a given model.
|
||||
|
|
@ -6080,7 +6093,7 @@ def get_max_tokens(model: str) -> int | None:
|
|||
|
||||
try:
|
||||
if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]:
|
||||
return litellm.model_cost[model]["max_output_tokens"]
|
||||
return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"])
|
||||
get_llm_provider: Final = litellm_utils.get_llm_provider
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(model=model)
|
||||
if custom_llm_provider == "huggingface":
|
||||
|
|
@ -6088,7 +6101,7 @@ def get_max_tokens(model: str) -> int | None:
|
|||
return max_tokens
|
||||
if model in litellm.model_cost: # check if extracted model is in model_list
|
||||
if "max_output_tokens" in litellm.model_cost[model]:
|
||||
return litellm.model_cost[model]["max_output_tokens"]
|
||||
return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"])
|
||||
else:
|
||||
raise Exception()
|
||||
return None
|
||||
|
|
@ -8236,8 +8249,9 @@ def trim_messages(
|
|||
try:
|
||||
if max_tokens is None:
|
||||
# Check if model is valid
|
||||
if model in litellm.model_cost:
|
||||
max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"]
|
||||
trimmed_row: Final = None if model is None else litellm.model_cost.get(model)
|
||||
max_tokens_for_model: Final = None if trimmed_row is None else trimmed_row.get("max_input_tokens")
|
||||
if isinstance(max_tokens_for_model, (int, float)):
|
||||
max_tokens = int(max_tokens_for_model * trim_ratio)
|
||||
else:
|
||||
# if user did not specify max (input) tokens
|
||||
|
|
@ -9054,11 +9068,9 @@ def validate_openai_optional_params(stop: str | list[str] | None = None, **kwarg
|
|||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_bundled_model_cost_map() -> dict[str, Any]:
|
||||
try:
|
||||
model_cost_path: Final = resources.files("litellm").joinpath("model_prices_and_context_window_backup.json")
|
||||
return json.loads(model_cost_path.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
return CATALOG.bundled().document()
|
||||
|
||||
|
||||
_get_bundled_model_cost_map = get_bundled_model_cost_map
|
||||
|
|
|
|||
|
|
@ -335,6 +335,7 @@ include = [
|
|||
"litellm/proxy/public_endpoints/whats_new.json",
|
||||
"litellm/proxy/common_utils/codex_base_instructions.md",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
|
||||
{ path = "model_prices_and_context_window.json", format = "sdist" },
|
||||
]
|
||||
exclude = [
|
||||
"litellm/proxy/enterprise",
|
||||
|
|
@ -357,6 +358,7 @@ cache-keys = [
|
|||
{ file = "litellm-rust/Cargo.lock" },
|
||||
{ file = "litellm-rust/Cargo.toml" },
|
||||
{ file = "litellm-rust/crates/**/*" },
|
||||
{ file = "model_prices_and_context_window.json" },
|
||||
]
|
||||
constraint-dependencies = [
|
||||
"tornado>=6.5.8",
|
||||
|
|
|
|||
|
|
@ -1,28 +1,30 @@
|
|||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import respx
|
||||
|
||||
import asyncio, httpx, importlib, os
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm import CustomLLM
|
||||
from litellm.litellm_core_utils import get_llm_provider_logic
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_llm_provider,
|
||||
resolve_model_and_provider_for_metadata,
|
||||
get_provider_affinity_header,
|
||||
inferred_provider,
|
||||
is_registered_custom_provider,
|
||||
resolve_model_and_provider_for_metadata,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from unittest.mock import patch
|
||||
|
||||
CUSTOM_PROVIDER: Final = "test-onprem-llm"
|
||||
|
||||
|
|
@ -634,15 +636,14 @@ def shipped_generalizations():
|
|||
added on this branch, so these tests install the rule the branch actually
|
||||
ships rather than depending on whatever the live URL returns.
|
||||
"""
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
get_fallback_generalization_rules,
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
previous = list(get_fallback_generalization_rules())
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
rules = backup.get("fallback_generalizations", {}).get("rules", [])
|
||||
rules = CATALOG.bundled().fallback_rules() or []
|
||||
set_fallback_generalizations(rules)
|
||||
try:
|
||||
yield rules
|
||||
|
|
|
|||
|
|
@ -4,16 +4,14 @@ count actual model entries, not reserved meta keys) and the extraction of the
|
|||
``fallback_generalizations`` block out of the raw map.
|
||||
"""
|
||||
|
||||
import asyncio, importlib, importlib.resources as importlib_get_model, json, litellm
|
||||
import asyncio, hashlib, importlib, json, litellm
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils import get_model_cost_map as get_model_cost_map_module
|
||||
|
|
@ -23,13 +21,11 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
match_routing_generalization,
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
from litellm._catalog import CATALOG, Catalog, Fetched, Unavailable
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
FALLBACK_GENERALIZATIONS_KEY,
|
||||
GetModelCostMap,
|
||||
_count_model_entries,
|
||||
_finalize_model_cost_map,
|
||||
get_model_cost_map_provenance,
|
||||
git_blob_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -39,75 +35,65 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def git_blob_id(body: bytes) -> str:
|
||||
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def _bundled_blob_id() -> str:
|
||||
path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json")
|
||||
path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json")
|
||||
with open(path, "rb") as f:
|
||||
return git_blob_id(f.read())
|
||||
|
||||
|
||||
def test_git_blob_id_is_what_git_hash_object_prints():
|
||||
assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987"
|
||||
def _parse(document: dict, min_model_count: int = 50, max_shrink_ratio: float = 0.5) -> Fetched | Unavailable:
|
||||
return Catalog.parse(
|
||||
json.dumps(document).encode(),
|
||||
etag=None,
|
||||
min_model_count=min_model_count,
|
||||
max_shrink_ratio=max_shrink_ratio,
|
||||
)
|
||||
|
||||
|
||||
def test_the_revision_is_what_git_hash_object_prints():
|
||||
parsed = Catalog.parse(
|
||||
b'{"gpt-5.4-mini": {"mode": "chat"}}\n', etag=None, min_model_count=0, max_shrink_ratio=0.0
|
||||
)
|
||||
assert isinstance(parsed, Fetched)
|
||||
assert parsed.revision == "18b9a8381e13a3b38a2128f184f631f95829e987"
|
||||
|
||||
|
||||
def test_the_bundled_revision_is_the_root_json_blob_id():
|
||||
assert CATALOG.bundled().revision == _bundled_blob_id()
|
||||
|
||||
|
||||
def _make_models(n: int) -> dict:
|
||||
return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)}
|
||||
|
||||
|
||||
def test_count_model_entries_excludes_reserved_keys():
|
||||
m = _make_models(3)
|
||||
m["sample_spec"] = {"foo": "bar"}
|
||||
m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
|
||||
assert _count_model_entries(m) == 3
|
||||
|
||||
|
||||
def test_validation_rejects_truly_shrunk_file_even_with_meta_keys():
|
||||
"""A file with only a handful of real models must be rejected as corrupt,
|
||||
and the extra meta keys must not inflate the count past the minimum."""
|
||||
"""Meta keys must not inflate the model count past the minimum"""
|
||||
shrunk = _make_models(5)
|
||||
shrunk["sample_spec"] = {"foo": "bar"}
|
||||
shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": [{"name": "x"}]}
|
||||
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=shrunk,
|
||||
backup_model_count=2000,
|
||||
min_model_count=50,
|
||||
)
|
||||
is False
|
||||
)
|
||||
assert isinstance(_parse(shrunk, max_shrink_ratio=0.0), Unavailable)
|
||||
|
||||
|
||||
def test_validation_accepts_healthy_file_with_meta_keys():
|
||||
healthy = _make_models(2000)
|
||||
healthy = _make_models(len(CATALOG.bundled().served()))
|
||||
healthy["sample_spec"] = {"foo": "bar"}
|
||||
healthy[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
|
||||
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=healthy,
|
||||
backup_model_count=2000,
|
||||
min_model_count=50,
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert isinstance(_parse(healthy), Fetched)
|
||||
|
||||
|
||||
def test_validation_rejects_significant_shrink_vs_backup():
|
||||
# 600 real models vs a 2000-model backup is below the 50% shrink threshold.
|
||||
shrunk = _make_models(600)
|
||||
shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=shrunk,
|
||||
backup_model_count=2000,
|
||||
min_model_count=50,
|
||||
max_shrink_ratio=0.5,
|
||||
)
|
||||
is False
|
||||
)
|
||||
def test_validation_rejects_significant_shrink_vs_the_bundled_map():
|
||||
shrunk = _make_models(len(CATALOG.bundled().served()) // 3)
|
||||
|
||||
assert isinstance(_parse(shrunk), Unavailable)
|
||||
|
||||
|
||||
def test_finalize_pops_key_and_installs_rules():
|
||||
def test_a_loaded_map_installs_its_rules_and_never_serves_the_block():
|
||||
previous = list(get_fallback_generalization_rules())
|
||||
try:
|
||||
raw = _make_models(2)
|
||||
|
|
@ -120,21 +106,24 @@ def test_finalize_pops_key_and_installs_rules():
|
|||
}
|
||||
]
|
||||
}
|
||||
finalized = _finalize_model_cost_map(raw)
|
||||
fetched = _parse(raw, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(fetched, Fetched)
|
||||
|
||||
# The reserved key is removed from the returned model map ...
|
||||
assert FALLBACK_GENERALIZATIONS_KEY not in finalized
|
||||
# ... and its rules are installed into the generalizations module.
|
||||
get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched))
|
||||
|
||||
assert FALLBACK_GENERALIZATIONS_KEY not in fetched.served()
|
||||
assert match_routing_generalization("widget-9") == "openai"
|
||||
finally:
|
||||
set_fallback_generalizations(previous)
|
||||
|
||||
|
||||
def test_finalize_with_no_block_clears_rules():
|
||||
def test_a_loaded_map_with_no_block_clears_rules():
|
||||
previous = list(get_fallback_generalization_rules())
|
||||
try:
|
||||
set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}])
|
||||
_finalize_model_cost_map(_make_models(2))
|
||||
fetched = _parse(_make_models(2), min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(fetched, Fetched)
|
||||
get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched))
|
||||
assert match_capability_generalizations("x-1") is None
|
||||
finally:
|
||||
set_fallback_generalizations(previous)
|
||||
|
|
@ -144,7 +133,7 @@ def test_shipped_backup_carries_the_claude_routing_rules():
|
|||
"""The bundled backup must ship the Claude routing rules so a fresh install
|
||||
(or an offline fallback) routes unknown Claude models without code changes.
|
||||
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().document()
|
||||
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
|
||||
names = [r.get("name") for r in rules]
|
||||
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
|
||||
|
|
@ -166,7 +155,7 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
|
|||
``bedrockz/anthropic.claude-...`` resolve to bedrock and slip through a
|
||||
``bedrock/*`` key, so every shipped routing rule must anchor to the start of
|
||||
the name and never match an id carrying an unrecognized namespace prefix."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().document()
|
||||
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
|
||||
|
||||
routing_rules = [r for r in rules if "litellm_provider" in r["model_info"]]
|
||||
|
|
@ -204,7 +193,7 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
|
|||
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
|
||||
carries only its delta; capability unioning stacks it onto the baseline, so the
|
||||
baseline block is never duplicated across rules and no rule needs ``extends``."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().document()
|
||||
|
||||
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
|
||||
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
|
||||
|
|
@ -314,7 +303,7 @@ async def test_refetch_retries_429_honoring_retry_after():
|
|||
sleeper = _SleepRecorder()
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client)
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert len(result.model_cost_map) > 100
|
||||
assert len(result.fetched.served()) > 100
|
||||
assert calls["count"] == 3
|
||||
assert sleeper.waits == [7.0, 7.0]
|
||||
|
||||
|
|
@ -384,6 +373,7 @@ async def test_refetch_invalid_json_fails_immediately():
|
|||
result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client)
|
||||
assert isinstance(result, ModelCostMapReloadUnavailable)
|
||||
assert "invalid JSON" in result.reason
|
||||
assert "unusable" in result.reason
|
||||
assert calls["count"] == 1
|
||||
assert sleeper.waits == []
|
||||
|
||||
|
|
@ -395,7 +385,7 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in():
|
|||
client, _calls = _mock_client([httpx.Response(200, content=tiny)])
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
assert isinstance(result, ModelCostMapReloadUnavailable)
|
||||
assert "integrity validation" in result.reason
|
||||
assert "below 0.5 of reference count" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -414,7 +404,7 @@ async def test_refetch_respects_local_env_override(monkeypatch, flag):
|
|||
client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)),
|
||||
)
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert len(result.model_cost_map) > 100
|
||||
assert len(result.fetched.served()) > 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -441,7 +431,7 @@ async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later
|
|||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
monkeypatch.setattr(litellm, "model_cost", result.model_cost_map)
|
||||
monkeypatch.setattr(litellm, "model_cost", result.fetched.served())
|
||||
litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False)
|
||||
assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048
|
||||
assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777
|
||||
|
|
@ -555,7 +545,7 @@ def test_get_model_cost_map_fetches_s3_catalog(monkeypatch: pytest.MonkeyPatch)
|
|||
for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"):
|
||||
monkeypatch.setattr(source_info, name, getattr(source_info, name))
|
||||
|
||||
s3_content: Final = _make_models(GetModelCostMap.get_backup_model_count())
|
||||
s3_content: Final = _make_models(len(CATALOG.bundled().served()))
|
||||
s3_client: Final = _FakeS3Client(json.dumps(s3_content).encode())
|
||||
monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: s3_client)
|
||||
|
||||
|
|
@ -623,8 +613,10 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
|
|||
for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"):
|
||||
monkeypatch.setattr(source_info, name, getattr(source_info, name))
|
||||
|
||||
held = litellm.model_cost
|
||||
remote_map = _load_root_cost_map()
|
||||
remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"}
|
||||
remote_map["gpt-4o"]["input_cost_per_token"] = 0.125
|
||||
client, calls = _mock_client(
|
||||
[
|
||||
httpx.ConnectError("connection refused"),
|
||||
|
|
@ -650,7 +642,7 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
|
|||
try:
|
||||
assert len(threads) == 1
|
||||
assert "claude-remote-only-test" not in cost_map
|
||||
assert cost_map.keys() == _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys()
|
||||
assert cost_map.keys() == CATALOG.bundled().served().keys()
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "local"
|
||||
assert source["fallback_reason"].startswith("Remote fetch failed:")
|
||||
|
|
@ -661,6 +653,8 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
|
|||
assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0
|
||||
assert calls["count"] == 2
|
||||
assert "claude-remote-only-test" in litellm.model_cost
|
||||
assert litellm.model_cost is held
|
||||
assert held["gpt-4o"]["input_cost_per_token"] == 0.125
|
||||
assert "claude-remote-only-test" in litellm.anthropic_models
|
||||
assert "my-runtime-model" in litellm.model_cost
|
||||
source = get_model_cost_map_source_info()
|
||||
|
|
@ -768,7 +762,8 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej
|
|||
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "local"
|
||||
assert source["fallback_reason"] == "Remote data failed integrity validation"
|
||||
assert source["fallback_reason"].startswith("Remote fetch failed:")
|
||||
assert "below minimum" in source["fallback_reason"]
|
||||
assert source["etag"] is None
|
||||
assert source["source_revision"] == _bundled_blob_id()
|
||||
assert source["source_revision"] != git_blob_id(shrunk_body)
|
||||
|
|
@ -862,119 +857,24 @@ _SCALAR_DEFAULTS = {
|
|||
"cohere_key": getattr(litellm, "cohere_key", None),
|
||||
}
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestCheckIsValidDict:
|
||||
"""Unit tests for _check_is_valid_dict."""
|
||||
|
||||
def test_should_reject_non_dict(self):
|
||||
"""Non-dict should fail."""
|
||||
assert GetModelCostMap._check_is_valid_dict("not a dict") is False
|
||||
@pytest.mark.parametrize(
|
||||
("body", "min_model_count", "max_shrink_ratio", "usable"),
|
||||
[
|
||||
pytest.param(b'"not a dict"', 10, 0.5, False, id="not_an_object"),
|
||||
pytest.param(b"{}", 10, 0.5, False, id="empty"),
|
||||
pytest.param(b"[1, 2, 3]", 10, 0.5, False, id="list"),
|
||||
pytest.param(b"null", 10, 0.5, False, id="null"),
|
||||
pytest.param(json.dumps(_make_models(5)).encode(), 10, 0.0, False, id="below_minimum"),
|
||||
pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.5, False, id="shrunk"),
|
||||
pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.0, True, id="no_ratio_floor"),
|
||||
pytest.param(json.dumps(_make_models(60)).encode(), 61, 0.0, False, id="one_below_minimum"),
|
||||
],
|
||||
)
|
||||
def test_parse_applies_mains_whole_map_checks(body, min_model_count, max_shrink_ratio, usable):
|
||||
parsed = Catalog.parse(body, etag=None, min_model_count=min_model_count, max_shrink_ratio=max_shrink_ratio)
|
||||
assert isinstance(parsed, Fetched) is usable
|
||||
|
||||
def test_should_reject_empty_dict(self):
|
||||
"""Empty dict should fail."""
|
||||
assert GetModelCostMap._check_is_valid_dict({}) is False
|
||||
|
||||
def test_should_reject_list(self):
|
||||
"""List should fail."""
|
||||
assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False
|
||||
|
||||
def test_should_reject_none(self):
|
||||
"""None should fail."""
|
||||
assert GetModelCostMap._check_is_valid_dict(None) is False
|
||||
|
||||
def test_should_accept_non_empty_dict(self):
|
||||
"""Non-empty dict should pass."""
|
||||
assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestCheckModelCountNotReduced:
|
||||
"""Unit tests for _check_model_count_not_reduced."""
|
||||
|
||||
def test_should_reject_too_few_models(self):
|
||||
"""Fetched map with fewer models than min_model_count should fail."""
|
||||
small_map = {f"model-{i}": {} for i in range(5)}
|
||||
assert (
|
||||
GetModelCostMap._check_model_count_not_reduced(
|
||||
fetched_map=small_map, backup_model_count=0, min_model_count=10
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
def test_should_reject_significant_shrinkage(self):
|
||||
"""Fetched map that shrunk >50% vs backup should fail."""
|
||||
fetched = {f"model-{i}": {} for i in range(40)} # 40% of 100
|
||||
assert (
|
||||
GetModelCostMap._check_model_count_not_reduced(
|
||||
fetched_map=fetched, backup_model_count=100, min_model_count=10
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
def test_should_accept_when_above_threshold(self):
|
||||
"""Fetched map at 60% of backup (above 50% threshold) should pass."""
|
||||
fetched = {f"model-{i}": {} for i in range(60)}
|
||||
assert (
|
||||
GetModelCostMap._check_model_count_not_reduced(
|
||||
fetched_map=fetched, backup_model_count=100, min_model_count=10
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
def test_should_accept_growth(self):
|
||||
"""Fetched map larger than backup should pass."""
|
||||
fetched = {f"model-{i}": {} for i in range(120)}
|
||||
assert (
|
||||
GetModelCostMap._check_model_count_not_reduced(
|
||||
fetched_map=fetched, backup_model_count=100, min_model_count=10
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
def test_should_accept_with_empty_backup(self):
|
||||
"""When backup is empty, only min_model_count matters."""
|
||||
fetched = {f"model-{i}": {} for i in range(15)}
|
||||
assert (
|
||||
GetModelCostMap._check_model_count_not_reduced(
|
||||
fetched_map=fetched, backup_model_count=0, min_model_count=10
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestValidateModelCostMap:
|
||||
"""Unit tests for validate_model_cost_map (combines both checks)."""
|
||||
|
||||
def test_should_reject_non_dict(self):
|
||||
"""Non-dict should fail at check 1."""
|
||||
assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_model_count=0) is False
|
||||
|
||||
def test_should_reject_empty_map(self):
|
||||
"""Empty dict should fail at check 1."""
|
||||
assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_model_count=0) is False
|
||||
|
||||
def test_should_reject_significant_shrinkage(self):
|
||||
"""Should fail at check 2 (shrinkage)."""
|
||||
fetched = {f"model-{i}": {} for i in range(40)}
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
|
||||
is False
|
||||
)
|
||||
|
||||
def test_should_accept_valid_map(self):
|
||||
"""Should pass both checks."""
|
||||
fetched = {f"model-{i}": {} for i in range(120)}
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
|
||||
is True
|
||||
)
|
||||
|
||||
def test_should_accept_equal_size_map(self):
|
||||
"""Equal size should pass both checks."""
|
||||
fetched = {f"model-{i}": {} for i in range(100)}
|
||||
assert (
|
||||
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
|
||||
is True
|
||||
)
|
||||
|
||||
def _fetch_with_single_outcome(monkeypatch, outcome):
|
||||
from litellm.litellm_core_utils import get_model_cost_map as module
|
||||
|
|
@ -989,7 +889,7 @@ def _fetch_with_single_outcome(monkeypatch, outcome):
|
|||
|
||||
|
||||
def _backup_keys():
|
||||
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys()
|
||||
return CATALOG.bundled().served().keys()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
|
|
@ -1028,7 +928,7 @@ class TestGetModelCostMapFallback:
|
|||
assert fetches == 1
|
||||
assert result.keys() == _backup_keys()
|
||||
assert source["source"] == "local"
|
||||
assert source["fallback_reason"] == f"Remote fetch failed: expected a JSON object from {_URL}, got list"
|
||||
assert source["fallback_reason"].startswith(f"Remote fetch failed: model cost map from {_URL} is unusable")
|
||||
|
||||
def test_should_fallback_when_fetched_map_shrinks_dramatically(self, monkeypatch):
|
||||
"""When upstream returns far fewer models than backup, should fall back."""
|
||||
|
|
@ -1039,7 +939,8 @@ class TestGetModelCostMapFallback:
|
|||
|
||||
assert fetches == 1
|
||||
assert result.keys() == _backup_keys()
|
||||
assert source == {**source, "source": "local", "fallback_reason": "Remote data failed integrity validation"}
|
||||
assert source["source"] == "local"
|
||||
assert "below minimum" in source["fallback_reason"]
|
||||
|
||||
def test_should_use_local_map_when_env_var_set(self):
|
||||
"""LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely."""
|
||||
|
|
@ -1056,14 +957,14 @@ class TestBackupModelCostMapExists:
|
|||
"""Validates the local backup file is always present and valid."""
|
||||
|
||||
def test_should_have_backup_file(self):
|
||||
"""The backup model cost map must exist and be loadable."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
"""The bundled model cost map must be embedded and loadable."""
|
||||
backup = CATALOG.bundled().served()
|
||||
assert isinstance(backup, dict)
|
||||
assert len(backup) > 0, "Backup model cost map is empty"
|
||||
|
||||
def test_should_have_minimum_models_in_backup(self):
|
||||
"""The backup must contain a reasonable number of models."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().served()
|
||||
assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100"
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
|
|
@ -1241,26 +1142,156 @@ def test_get_model_cost_map():
|
|||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred: {e}")
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
"setup_and_teardown_local_testing",
|
||||
)
|
||||
def test_get_backup_model_cost_map():
|
||||
with importlib_get_model.open_text("litellm", "model_prices_and_context_window_backup.json") as f:
|
||||
print("inside backup")
|
||||
content = json.load(f)
|
||||
print("content", content)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backup", [b'["only-in-the-backup"]', b'"only-in-the-backup"', b"null"])
|
||||
def test_a_bundled_backup_that_is_not_a_json_object_is_rejected_without_quoting_it(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, backup: bytes
|
||||
):
|
||||
(tmp_path / "model_prices_and_context_window_backup.json").write_bytes(backup)
|
||||
monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Install through the Rust catalog (decisions 1, 15, 23 and 25)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
with pytest.raises(ValidationError) as rejected:
|
||||
GetModelCostMap.load_local_model_cost_map_with_revision()
|
||||
import logging
|
||||
import types
|
||||
|
||||
assert "only-in-the-backup" not in str(rejected.value)
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _bundled_document() -> dict:
|
||||
return CATALOG.bundled().document()
|
||||
|
||||
|
||||
def _remote(edit) -> Fetched:
|
||||
document = json.loads(json.dumps(_bundled_document()))
|
||||
edit(document)
|
||||
parsed = Catalog.parse(json.dumps(document).encode(), etag=None, min_model_count=50, max_shrink_ratio=0.5)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return parsed
|
||||
|
||||
|
||||
def _bound_catalog() -> tuple[types.ModuleType, Catalog]:
|
||||
module = types.ModuleType("fake_litellm")
|
||||
module.model_cost = {}
|
||||
catalog = Catalog.bound_to(module, "model_cost")
|
||||
catalog.install(catalog.bundled())
|
||||
return module, catalog
|
||||
|
||||
|
||||
def test_sample_spec_is_not_served_but_stays_in_the_bundled_document():
|
||||
assert "sample_spec" not in litellm.model_cost
|
||||
assert "sample_spec" in CATALOG.bundled().document()
|
||||
|
||||
|
||||
def test_a_fetched_row_that_fails_validation_is_replaced_by_bundled_or_left_out(caplog):
|
||||
module, catalog = _bound_catalog()
|
||||
bundled_gpt_4o = dict(module.model_cost["gpt-4o"])
|
||||
|
||||
def edit(document):
|
||||
document["gpt-4o"]["input_cost_per_token"] = -1
|
||||
document["acme/new"] = {"litellm_provider": "acme", "input_cost_per_token": "free"}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
installed = get_model_cost_map_module._install( # pyright: ignore[reportPrivateUsage] # only the install step logs rejected rows
|
||||
_remote(edit), catalog
|
||||
)
|
||||
|
||||
assert module.model_cost["gpt-4o"] == bundled_gpt_4o
|
||||
assert "acme/new" not in module.model_cost
|
||||
assert {error.key for error in installed.rejected} == {"gpt-4o", "acme/new"}
|
||||
warnings = [record for record in caplog.records if "failed validation" in record.getMessage()]
|
||||
assert len(warnings) == 1
|
||||
assert "gpt-4o" in warnings[0].getMessage() and "acme/new" in warnings[0].getMessage()
|
||||
|
||||
|
||||
def test_a_field_the_fetched_row_lacks_is_served_from_bundled():
|
||||
module, catalog = _bound_catalog()
|
||||
bundled_max_output_tokens = module.model_cost["gpt-4o"]["max_output_tokens"]
|
||||
|
||||
catalog.install(_remote(lambda document: document["gpt-4o"].pop("max_output_tokens")))
|
||||
|
||||
assert module.model_cost["gpt-4o"]["max_output_tokens"] == bundled_max_output_tokens
|
||||
|
||||
|
||||
def test_a_key_the_fetched_map_lacks_stays_until_the_map_writes_it_as_null():
|
||||
module, catalog = _bound_catalog()
|
||||
|
||||
catalog.install(_remote(lambda document: document.pop("gpt-4o")))
|
||||
assert "gpt-4o" in module.model_cost
|
||||
|
||||
catalog.install(_remote(lambda document: document.update({"gpt-4o": None})))
|
||||
assert "gpt-4o" not in module.model_cost
|
||||
|
||||
|
||||
def test_a_null_field_in_the_fetched_map_removes_it():
|
||||
module, catalog = _bound_catalog()
|
||||
|
||||
catalog.install(_remote(lambda document: document["gpt-4o"].update({"max_output_tokens": None})))
|
||||
|
||||
assert "max_output_tokens" not in module.model_cost["gpt-4o"]
|
||||
|
||||
|
||||
def _direct_writes(model_cost: dict) -> None:
|
||||
model_cost["gpt-4o"]["input_cost_per_token"] = 1.0
|
||||
model_cost["my-own-model"] = {"litellm_provider": "openai", "input_cost_per_token": 3.0}
|
||||
model_cost["gpt-4o-mini"] = {"litellm_provider": "openai", "input_cost_per_token": 4.0}
|
||||
del model_cost["gpt-4.1"]
|
||||
|
||||
|
||||
def _changed_upstream(document: dict) -> None:
|
||||
document["gpt-4o"]["input_cost_per_token"] = 9.0
|
||||
document["gpt-4o-mini"]["input_cost_per_token"] = 9.0
|
||||
document["gpt-4o-mini"]["max_input_tokens"] = 1
|
||||
|
||||
|
||||
def test_direct_writes_survive_a_reload():
|
||||
module, catalog = _bound_catalog()
|
||||
held = module.model_cost
|
||||
_direct_writes(module.model_cost)
|
||||
|
||||
catalog.install(_remote(_changed_upstream))
|
||||
|
||||
assert module.model_cost is held
|
||||
assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 1.0
|
||||
assert module.model_cost["my-own-model"]["input_cost_per_token"] == 3.0
|
||||
assert module.model_cost["gpt-4o-mini"]["input_cost_per_token"] == 4.0
|
||||
assert "max_input_tokens" not in module.model_cost["gpt-4o-mini"]
|
||||
assert "gpt-4.1" not in module.model_cost
|
||||
|
||||
|
||||
def test_rebinding_the_dict_then_reloading_drops_direct_writes():
|
||||
module, catalog = _bound_catalog()
|
||||
_direct_writes(module.model_cost)
|
||||
module.model_cost = dict(module.model_cost)
|
||||
|
||||
catalog.install(_remote(_changed_upstream))
|
||||
|
||||
assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 9.0
|
||||
assert "my-own-model" not in module.model_cost
|
||||
assert module.model_cost["gpt-4o-mini"]["max_input_tokens"] == 1
|
||||
assert "gpt-4.1" in module.model_cost
|
||||
|
||||
|
||||
def test_install_keeps_a_row_object_whose_fields_did_not_change():
|
||||
module, catalog = _bound_catalog()
|
||||
row = module.model_cost["gpt-4o"]
|
||||
|
||||
catalog.install(_remote(lambda document: document["gpt-4o-mini"].update({"input_cost_per_token": 9.0})))
|
||||
|
||||
assert module.model_cost["gpt-4o"] is row
|
||||
|
||||
|
||||
def test_install_refuses_a_bound_attribute_that_is_not_a_dict():
|
||||
module, catalog = _bound_catalog()
|
||||
module.model_cost = types.MappingProxyType({})
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
catalog.install(catalog.bundled())
|
||||
|
||||
|
||||
def test_the_local_map_is_the_embedded_one_not_the_backup_file(monkeypatch, tmp_path):
|
||||
(tmp_path / "model_prices_and_context_window_backup.json").write_text(
|
||||
json.dumps({"only-in-the-backup": {"litellm_provider": "openai"}})
|
||||
)
|
||||
monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path, raising=False)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
cost_map = get_model_cost_map(url=_URL)
|
||||
|
||||
assert "only-in-the-backup" not in cost_map
|
||||
assert cost_map.keys() == CATALOG.bundled().served().keys()
|
||||
|
|
|
|||
|
|
@ -241,21 +241,15 @@ print("compatible")
|
|||
|
||||
|
||||
@pytest.mark.parametrize("rust", (None, "0", "1"))
|
||||
def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None:
|
||||
def test_tokenization_stays_offline(tmp_path: Path, rust: str | None) -> None:
|
||||
script: Final = """
|
||||
import importlib.abc
|
||||
import sys
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
import litellm
|
||||
def reject_network(event, args):
|
||||
if event == "socket.connect":
|
||||
raise AssertionError("tokenizer attempted a network connection")
|
||||
sys.addaudithook(reject_network)
|
||||
class Block(importlib.abc.MetaPathFinder):
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname == "litellm.rust_bridge._native":
|
||||
raise ImportError("native extension is unavailable")
|
||||
sys.meta_path.insert(0, Block())
|
||||
import litellm
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
import tiktoken
|
||||
from tokenizers import Tokenizer
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import httpx
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
|
||||
from litellm.llms.azure.azure import AzureChatCompletion
|
||||
from litellm.llms.azure.image_generation import get_azure_image_generation_config
|
||||
|
|
@ -20,7 +20,7 @@ from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_par
|
|||
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_local_model_cost_map(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
|
||||
litellm.get_model_info.cache_clear()
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
yield
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ the sharded CI (coverage collection runs against this tree).
|
|||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
|
|
@ -632,10 +632,10 @@ class TestAgentCoreSearchEdgeCases:
|
|||
def test_search_cost_lookup_is_mapped(self, monkeypatch):
|
||||
"""Assert against the map in this checkout: the remote cost map litellm loads by
|
||||
default only carries providers already released."""
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.search.cost_calculator import search_provider_cost_per_query
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
|
||||
price_per_query = litellm.model_cost["agentcore/search"]["input_cost_per_query"]
|
||||
assert price_per_query > 0
|
||||
assert search_provider_cost_per_query(
|
||||
|
|
|
|||
|
|
@ -1,9 +1,16 @@
|
|||
|
||||
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
|
||||
import asyncio
|
||||
import importlib
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.bedrock.common_utils import(
|
||||
import litellm
|
||||
import litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map
|
||||
from litellm import completion
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
AmazonBedrockGlobalConfig,
|
||||
BedrockModelInfo,
|
||||
bedrock_native_output_is_opt_in,
|
||||
|
|
@ -16,11 +23,8 @@ from litellm.llms.bedrock.common_utils import(
|
|||
strip_bedrock_throughput_suffix,
|
||||
without_bedrock_native_output_opt_in,
|
||||
)
|
||||
from collections.abc import Iterator
|
||||
from litellm import completion
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# get_bedrock_response_stream_shape lazy-load tests #
|
||||
|
|
@ -33,10 +37,8 @@ def _reset_bedrock_response_stream_shape_cache():
|
|||
import litellm.llms.bedrock.common_utils as mod
|
||||
|
||||
mod.get_bedrock_response_stream_shape.cache_clear()
|
||||
mod._get_local_model_cost_map.cache_clear()
|
||||
yield
|
||||
mod.get_bedrock_response_stream_shape.cache_clear()
|
||||
mod._get_local_model_cost_map.cache_clear()
|
||||
|
||||
|
||||
def test_bedrock_response_stream_shape_lazy_loads_once():
|
||||
|
|
@ -342,9 +344,9 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch)
|
|||
def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(
|
||||
model, expected_ceiling
|
||||
):
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
model_info = GetModelCostMap.load_local_model_cost_map()[model]
|
||||
model_info = CATALOG.bundled().served()[model]
|
||||
|
||||
assert model_info["bedrock_output_config_effort_ceiling"] == expected_ceiling
|
||||
|
||||
|
|
@ -2034,6 +2036,7 @@ class TestBedrockGovCloudSupport:
|
|||
@pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"])
|
||||
def test_event_decoder_reports_only_missing_aws_dependency(missing):
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase
|
||||
|
||||
failure = ModuleNotFoundError("missing dependency", name=missing)
|
||||
|
|
|
|||
|
|
@ -7,12 +7,11 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations.
|
|||
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
import litellm.utils
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
|
||||
|
||||
|
||||
|
|
@ -681,7 +680,7 @@ class TestKimiK26ModelRegistry:
|
|||
@pytest.fixture(autouse=True)
|
||||
def model_cost_map(self):
|
||||
"""Load directly from the bundled backup so tests don't depend on remote fetch."""
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
return CATALOG.bundled().served()
|
||||
|
||||
|
||||
class TestMoonshotResponseSchemaSupport:
|
||||
|
|
@ -703,7 +702,7 @@ class TestMoonshotResponseSchemaSupport:
|
|||
|
||||
@pytest.fixture(autouse=True)
|
||||
def model_cost_map(self):
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
return CATALOG.bundled().served()
|
||||
|
||||
|
||||
class TestMoonshotReasoningEffort:
|
||||
|
|
@ -713,7 +712,7 @@ class TestMoonshotReasoningEffort:
|
|||
|
||||
@pytest.fixture(autouse=True)
|
||||
def force_local_model_cost(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
|
||||
|
||||
@pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"])
|
||||
def test_reasoning_model_supports_reasoning_effort(self, model):
|
||||
|
|
|
|||
|
|
@ -64,6 +64,15 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset(
|
|||
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",)
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
|
||||
class _RecordingRedisClient(Redis):
|
||||
def __init__(self) -> None:
|
||||
self.published: List[Tuple[str, str]] = []
|
||||
|
|
@ -860,7 +869,7 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None:
|
|||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=ModelCostMapReloaded(model_cost_map={"gpt-5.2": {"input_cost_per_token": 0.001}})
|
||||
return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}})
|
||||
),
|
||||
):
|
||||
await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,15 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
|
||||
def _attach_litellm_config(mock_prisma):
|
||||
"""Attach a litellm_config table mock (not in conftest's _PRISMA_TABLES)."""
|
||||
table = MagicMock()
|
||||
|
|
@ -55,7 +64,7 @@ def _attach_litellm_config(mock_prisma):
|
|||
|
||||
def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
||||
"""Admin can trigger a manual reload; handler returns model count + status."""
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
|
|
@ -65,7 +74,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}}
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fake_cost_map)),
|
||||
AsyncMock(return_value=_reloaded(fake_cost_map)),
|
||||
)
|
||||
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
|
||||
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
|
||||
|
|
@ -83,11 +92,12 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.post("/reload/model_cost_map")
|
||||
assert response.status_code == 200
|
||||
served = len(CATALOG.bundled().served().keys() | fake_cost_map.keys())
|
||||
body = normalize(response.json(), volatile=_VOLATILE)
|
||||
assert body == {
|
||||
"message": "Price data reloaded successfully! 2 models updated.",
|
||||
"message": f"Price data reloaded successfully! {served} models updated.",
|
||||
"status": "success",
|
||||
"models_count": 2,
|
||||
"models_count": served,
|
||||
"timestamp": "<VOLATILE>",
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
|
|
@ -103,7 +113,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
|
|||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _finalize_model_cost_map, git_blob_id
|
||||
import hashlib
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
|
|
@ -111,7 +121,10 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
|
|||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
|
||||
body = _ROOT_COST_MAP.read_bytes()
|
||||
expected = {"source_revision": git_blob_id(body), "etag": _SERVED_ETAG}
|
||||
expected = {
|
||||
"source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(),
|
||||
"etag": _SERVED_ETAG,
|
||||
}
|
||||
served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body)
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
|
||||
|
|
@ -142,7 +155,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
|
|||
assert {key: status_response.json()[key] for key in expected} == expected
|
||||
assert public_response.status_code == 200
|
||||
assert "gpt-4o" in public_response.json()
|
||||
assert reload_body["models_count"] == len(_finalize_model_cost_map(json.loads(body)))
|
||||
assert reload_body["models_count"] == len(litellm.model_cost)
|
||||
|
||||
|
||||
def test_reload_model_cost_map_fetch_failure_502_keeps_map(
|
||||
|
|
|
|||
|
|
@ -80,6 +80,15 @@ example_embedding_result = {
|
|||
}
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
|
||||
def mock_patch_aembedding():
|
||||
return mock.patch(
|
||||
"litellm.proxy.proxy_server.llm_router.aembedding",
|
||||
|
|
@ -4551,7 +4560,7 @@ class TestPriceDataReloadAPI:
|
|||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=ModelCostMapReloaded(model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
|
||||
return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
|
||||
),
|
||||
):
|
||||
# Mock the database connection
|
||||
|
|
@ -4569,8 +4578,11 @@ class TestPriceDataReloadAPI:
|
|||
assert "timestamp" in data
|
||||
assert "models_count" in data
|
||||
# The new implementation immediately reloads and returns the count
|
||||
assert "Price data reloaded successfully! 1 models updated." in data["message"]
|
||||
assert data["models_count"] == 1
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
served = len(CATALOG.bundled().served())
|
||||
assert f"Price data reloaded successfully! {served} models updated." in data["message"]
|
||||
assert data["models_count"] == served
|
||||
finally:
|
||||
# Restore the full model cost map so subsequent tests are not affected
|
||||
litellm.model_cost = original_model_cost
|
||||
|
|
@ -4822,7 +4834,7 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=mock_cost_map)),
|
||||
new=AsyncMock(return_value=_reloaded(mock_cost_map)),
|
||||
):
|
||||
# Mock the database connection
|
||||
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
|
||||
|
|
@ -4895,13 +4907,12 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
||||
assert litellm.model_cost["gpt-3.5-turbo"] == {"input_cost_per_token": 0.001}
|
||||
assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.001
|
||||
assert proxy_config.model_cost_map_loaded_at == frozen_now
|
||||
assert mock_prisma.db.litellm_config.update_many.call_args[1] == {
|
||||
"data": {"last_run_at": frozen_now},
|
||||
|
|
@ -4980,13 +4991,12 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4-test": {"input_cost_per_token": 0.5}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
||||
assert litellm.model_cost["gpt-4-test"] == {"input_cost_per_token": 0.5}
|
||||
assert litellm.model_cost["gpt-4-test"]["input_cost_per_token"] == 0.5
|
||||
assert proxy_config.model_cost_map_loaded_at == frozen_now
|
||||
assert mock_prisma.db.litellm_config.update_many.call_args[1]["data"] == {"last_run_at": frozen_now}
|
||||
|
||||
|
|
@ -5025,8 +5035,7 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
for _ in range(3):
|
||||
|
|
@ -5069,8 +5078,7 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5105,8 +5113,7 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5147,8 +5154,7 @@ class TestPriceDataReloadIntegration:
|
|||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4": {"input_cost_per_token": 0.1}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5236,8 +5242,7 @@ class TestPriceDataReloadIntegration:
|
|||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
|
||||
return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
|
||||
)
|
||||
),
|
||||
),
|
||||
|
|
@ -5271,9 +5276,9 @@ class TestPriceDataReloadIntegration:
|
|||
model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}
|
||||
)
|
||||
|
||||
models_count = _swap_in_model_cost_map({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}})
|
||||
models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched)
|
||||
|
||||
assert models_count == 1
|
||||
assert models_count == len(litellm.model_cost) - 1
|
||||
assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
|
|
@ -5311,8 +5316,7 @@ class TestPriceDataReloadIntegration:
|
|||
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = ModelCostMapReloaded(
|
||||
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
mock_prisma.db.litellm_config.upsert = AsyncMock(
|
||||
return_value=_reload_schedule_row({}, reload_revision=9)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.search.cost_calculator import search_provider_cost_per_query
|
||||
|
|
@ -29,7 +30,7 @@ SEARCH_MODE_PRICES: Final = {
|
|||
|
||||
@pytest.fixture
|
||||
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -64,7 +64,7 @@ FABLE_5_1_VARIANTS = (
|
|||
|
||||
|
||||
def test_fable_5_1_present_in_bundled_backup():
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().served()
|
||||
root = _load_root_cost_map()
|
||||
for model_name in FABLE_5_1_VARIANTS:
|
||||
assert model_name in backup, f"Missing from backup cost map: {model_name}"
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from typing import Final, cast
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
|
@ -74,7 +74,7 @@ def test_haiku_5_5_rows_allow_disabling_thinking_and_forced_tools(
|
|||
model_name: str,
|
||||
) -> None:
|
||||
root: Final = _load_root_cost_map()
|
||||
backup: Final = GetModelCostMap.load_local_model_cost_map()
|
||||
backup: Final = CATALOG.bundled().served()
|
||||
assert model_name in root
|
||||
row: Final = root[model_name]
|
||||
# https://platform.claude.com/docs/en/models/haiku-5-5/whats-new-haiku-5-5 (2026-10-07):
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -82,7 +82,7 @@ OPUS_5_5_VARIANTS = (
|
|||
|
||||
@pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS)
|
||||
def test_opus_5_5_present_in_bundled_backup(model_name):
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().served()
|
||||
root = _load_root_cost_map()
|
||||
assert model_name in backup
|
||||
assert model_name in root
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -60,7 +60,7 @@ SONNET_5_5_VARIANTS = (
|
|||
|
||||
@pytest.mark.parametrize("model_name", SONNET_5_5_VARIANTS)
|
||||
def test_sonnet_5_5_present_in_bundled_backup(model_name):
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
backup = CATALOG.bundled().served()
|
||||
root = _load_root_cost_map()
|
||||
assert model_name in backup
|
||||
assert model_name in root
|
||||
|
|
|
|||
|
|
@ -186,16 +186,32 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
|
|||
)
|
||||
|
||||
|
||||
_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it"
|
||||
_REMOVED_VISION: Final = (
|
||||
"openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it"
|
||||
)
|
||||
|
||||
|
||||
def test_bot_may_not_remove_models() -> None:
|
||||
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
|
||||
assert _failures(head, bot=False) == ()
|
||||
assert _failures(head) == ("bot PRs may not remove models: openrouter/b",)
|
||||
assert _failures(head, bot=False) == (_REMOVED_B,)
|
||||
assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b")
|
||||
|
||||
|
||||
def test_bot_may_not_remove_fields() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": _entry()})
|
||||
assert _failures(head, bot=False) == (_REMOVED_VISION,)
|
||||
assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision")
|
||||
|
||||
|
||||
def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None})
|
||||
assert _failures(head, bot=False) == ()
|
||||
|
||||
|
||||
def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}})
|
||||
assert _failures(head, bot=False) == ()
|
||||
assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",)
|
||||
|
||||
|
||||
def test_bot_may_not_change_special_root_keys() -> None:
|
||||
|
|
@ -244,7 +260,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co
|
|||
("head_ref", "expected_code", "expected_line"),
|
||||
[
|
||||
(BOT_REF, 1, "- bot PRs may not remove models: openrouter/b"),
|
||||
("litellm_fix_pricing", 0, "cost map guard passed (human PR, file checks only)"),
|
||||
("litellm_fix_pricing", 1, f"- {_REMOVED_B}"),
|
||||
],
|
||||
)
|
||||
def test_main_reads_both_revisions_from_git(
|
||||
|
|
|
|||
|
|
@ -1,14 +1,28 @@
|
|||
"""
|
||||
Tests for the ``aliases`` feature in the model cost map.
|
||||
|
||||
The ``_expand_model_aliases`` function processes ``aliases`` lists from model
|
||||
entries, creating shared dict references for alias entries at load time.
|
||||
The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
import json
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases
|
||||
from litellm._catalog import Catalog, Fetched, Unavailable
|
||||
|
||||
|
||||
def _parsed(model_cost: dict) -> Fetched | Unavailable:
|
||||
return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
|
||||
|
||||
def _expand(model_cost: dict) -> dict:
|
||||
parsed = _parsed(model_cost)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return parsed.served()
|
||||
|
||||
|
||||
def _error_kinds(model_cost: dict) -> list[str]:
|
||||
parsed = _parsed(model_cost)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return [error.kind for error in parsed.errors()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -29,7 +43,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert "my-model-20250101" in result
|
||||
assert result["my-model-20250101"]["input_cost_per_token"] == 1e-06
|
||||
|
|
@ -45,7 +59,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert "provider/model-v1" in result
|
||||
assert "provider/model-v2" in result
|
||||
|
|
@ -60,7 +74,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert result["alias-model"] is result["canonical-model"]
|
||||
|
||||
|
|
@ -74,7 +88,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert "aliases" not in result["my-model"]
|
||||
assert "aliases" not in result["my-model-alias"]
|
||||
|
|
@ -88,7 +102,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert "plain-model" in result
|
||||
assert result["plain-model"]["input_cost_per_token"] == 3e-06
|
||||
|
|
@ -104,7 +118,7 @@ class TestExpandModelAliases:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "model-a" in result
|
||||
|
|
@ -134,15 +148,11 @@ class TestAliasConflicts:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
with patch.object(verbose_logger, "warning") as mock_warn:
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
# The canonical "model-dated" entry is preserved, not overwritten
|
||||
assert "model-dated" in result
|
||||
# Verify a warning about the alias conflict was logged
|
||||
mock_warn.assert_called()
|
||||
warning_messages = " ".join(str(c) for c in mock_warn.call_args_list)
|
||||
assert "alias conflict" in warning_messages.lower()
|
||||
assert "alias_shadows_model" in _error_kinds(model_cost)
|
||||
|
||||
def test_duplicate_alias_across_entries(self):
|
||||
"""Same alias claimed by two different entries: second one is skipped."""
|
||||
|
|
@ -160,16 +170,12 @@ class TestAliasConflicts:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
with patch.object(verbose_logger, "warning") as mock_warn:
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
# "shared-alias" should point to model-a (first one wins)
|
||||
assert "shared-alias" in result
|
||||
assert result["shared-alias"]["input_cost_per_token"] == 1e-06
|
||||
# Verify a warning about the alias conflict was logged
|
||||
mock_warn.assert_called()
|
||||
warning_messages = " ".join(str(c) for c in mock_warn.call_args_list)
|
||||
assert "alias conflict" in warning_messages.lower()
|
||||
assert "alias_taken" in _error_kinds(model_cost)
|
||||
|
||||
def test_canonical_entry_not_overwritten_by_alias(self):
|
||||
"""An alias must never overwrite an existing canonical entry's data."""
|
||||
|
|
@ -187,7 +193,7 @@ class TestAliasConflicts:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
# Original entry must be preserved
|
||||
assert result["existing-model"]["input_cost_per_token"] == original_cost
|
||||
|
|
@ -211,7 +217,7 @@ class TestAliasIntegration:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
# Mutate via alias
|
||||
result["alias"]["input_cost_per_token"] = 999
|
||||
|
|
@ -232,7 +238,7 @@ class TestAliasIntegration:
|
|||
"mode": "chat",
|
||||
},
|
||||
}
|
||||
result = _expand_model_aliases(model_cost)
|
||||
result = _expand(model_cost)
|
||||
|
||||
assert len(result) == 4 # 2 canonical + 2 aliases
|
||||
assert "alias-1" in result
|
||||
|
|
@ -240,6 +246,5 @@ class TestAliasIntegration:
|
|||
assert "plain-model" in result
|
||||
assert "model-with-alias" in result
|
||||
|
||||
def test_expand_on_empty_dict(self):
|
||||
"""Expanding an empty dict returns an empty dict."""
|
||||
assert _expand_model_aliases({}) == {}
|
||||
def test_an_empty_map_is_unavailable(self):
|
||||
assert isinstance(_parsed({}), Unavailable)
|
||||
|
|
|
|||
|
|
@ -1384,13 +1384,6 @@ ALIAS_CASES: Final = (
|
|||
"is_non_openai_azure_model",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"litellm.litellm_core_utils.get_model_cost_map",
|
||||
"GetModelCostMap",
|
||||
"_get_backup_model_count",
|
||||
"get_backup_model_count",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"litellm.litellm_core_utils.health_check_helpers",
|
||||
"HealthCheckHelpers",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ calculations for DB-sourced models with prompt caching pricing.
|
|||
|
||||
import copy
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final, NoReturn
|
||||
|
|
@ -19,11 +20,18 @@ import pytest
|
|||
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
def _fetched(cost_map: Mapping[str, object]) -> Fetched:
|
||||
parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return parsed
|
||||
|
||||
|
||||
def _snapshot_model_cost_entries(keys):
|
||||
return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys}
|
||||
|
||||
|
|
@ -1038,11 +1046,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
|
|||
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
|
||||
}
|
||||
real_lookup: Final = litellm.utils.get_model_info
|
||||
reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered
|
||||
|
||||
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
|
||||
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
|
||||
if litellm.model_cost is not reloaded_cost_map:
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
if not reloads:
|
||||
reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map)))
|
||||
return info
|
||||
|
||||
with pytest.MonkeyPatch.context() as lookup_patch:
|
||||
|
|
@ -1056,52 +1065,6 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model"
|
||||
deployment_key: Final = f"bedrock/{catalog_key}"
|
||||
catalog_row: Final = {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 3.3e-06,
|
||||
"output_cost_per_token": 1.65e-05,
|
||||
}
|
||||
reloaded_input_cost: Final = 6.6e-06
|
||||
registered_output_cost: Final = 2e-05
|
||||
builtin_cost_map: Final = litellm.get_model_cost_map(url="")
|
||||
reloaded_cost_map: Final = {
|
||||
**builtin_cost_map,
|
||||
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
|
||||
}
|
||||
|
||||
class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]):
|
||||
def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping
|
||||
if litellm.model_cost is self:
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
super().update(other)
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}),
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
litellm.register_model(
|
||||
{deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
|
||||
try:
|
||||
assert litellm.model_cost is reloaded_cost_map
|
||||
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
|
||||
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_register_model_leaves_the_entry_a_lookup_is_walking_intact(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
catalog_key: Final = "lit-5853-live-entry-model"
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
|
@ -1139,21 +1102,14 @@ _ALIAS_KEY: Final = "lit-5853-alias-model"
|
|||
|
||||
def _install_aliased_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
aliased: Final = AliasedCostEntry(
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06},
|
||||
(_CANONICAL_KEY, _ALIAS_KEY),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
_expand_model_aliases(
|
||||
{
|
||||
**litellm.get_model_cost_map(url=""),
|
||||
_CANONICAL_KEY: {
|
||||
"aliases": [_ALIAS_KEY],
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
},
|
||||
}
|
||||
),
|
||||
{**litellm.get_model_cost_map(url=""), _CANONICAL_KEY: aliased, _ALIAS_KEY: aliased},
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
|
|
|||
|
|
@ -20337,9 +20337,9 @@ class TestTierParamsTheTargetAccepts:
|
|||
|
||||
@pytest.fixture(autouse=True)
|
||||
def force_local_model_cost(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
|
||||
|
||||
@staticmethod
|
||||
def _router(model: str) -> litellm.Router:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue