From fb6b845920ff76417bee67fef33a44b5b0367dac Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:58:05 +0000 Subject: [PATCH] feat(python-bridge): bind model registry and snapshot layers --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/python-bridge/AGENTS.md | 1 + litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/lib.rs | 40 +- .../python-bridge/src/model_registry.rs | 977 ++++++++++++++++++ litellm/rust_bridge/_native.pyi | 77 +- 6 files changed, 1095 insertions(+), 2 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/model_registry.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index e60cb676cc5..6f119e5395d 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4333,6 +4333,7 @@ dependencies = [ "litellm-inference-transcription", "litellm-llms", "litellm-llms-types", + "litellm-model-catalog", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index c435ba37e78..0286a5b2cd9 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -116,6 +116,7 @@ GIL handling to `litellm-host-python`. ## Tests +- The model catalog exports have no module-missing fallback, because `import litellm` needs the catalog - `cargo test --workspace` must compile this crate. - Python tests must cover bridge disabled, bridge enabled, and module-missing fallback behavior for every exposed route. diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 6fc05c43146..0bc44272f20 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -24,6 +24,7 @@ litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true +litellm-model-catalog.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12827b76ff3..5c689b53bf9 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -8,6 +8,7 @@ mod execution; mod http; mod logger; mod marshal; +mod model_registry; mod preflight; mod python_settings; mod routes; @@ -30,6 +31,10 @@ mod _native { #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] + use crate::model_registry::{ + ModelCostLayers, ModelRegistry, RowError, model_row_errors, write_rows, + }; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -66,7 +71,17 @@ mod _native { dict.set_item( "_SecretManagerRuntime", py.get_type::(), - ) + )?; + let typing = py.import("typing")?; + let literal = typing.getattr("Literal")?; + let kinds = pyo3::types::PyTuple::new( + py, + ::VARIANTS + .iter() + .map(|kind| literal.get_item(<&'static str>::from(*kind))) + .collect::>>()?, + )?; + module.add("RowErrorKind", typing.getattr("Union")?.get_item(kinds)?) } } @@ -81,12 +96,35 @@ pub(crate) fn native_module(py: Python<'_>) -> Bound<'_, PyModule> { mod tests { use super::*; + #[rstest::rstest] + fn registry_rows_install_without_replacing_the_live_dict() { + Python::initialize(); + Python::attach(|py| { + let scope = pyo3::types::PyDict::new(py); + scope.set_item("native", native_module(py)).unwrap(); + py.run(pyo3::ffi::c_str!(r#" +registry = native.ModelRegistry.parse(b'{"model": {"litellm_provider": "openai", "input_cost_per_token": 0}}') +live = {"old": {}} +alias = live +assert native.write_rows(live, registry.served(), ["old", "missing"]) is None +assert live is alias +assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token": 0}} +"#), Some(&scope), Some(&scope)).unwrap(); + }); + } + #[rstest::rstest] fn module_registration_preserves_the_public_surface() { Python::initialize(); Python::attach(|py| { let mut expected = vec![ "NativeCacheHandle", + "ModelRegistry", + "ModelCostLayers", + "RowError", + "RowErrorKind", + "model_row_errors", + "write_rows", "RustBridgeDeclined", "RustUpstreamError", "ForkedAfterNativeRuntimeStarted", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs new file mode 100644 index 00000000000..37147eb1bae --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -0,0 +1,977 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::{Arc, LazyLock, Mutex, PoisonError}; + +use litellm_host_python::{from_py, release_gil, to_py}; +use litellm_model_catalog::wire::Fields; +use litellm_model_catalog::{ + ApplicationWrite, BUNDLED_REGISTRY, Catalog, Error, IntegrityLimits, Layers, Located, + SparseRow, diff_row, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::pybacked::PyBackedBytes; +use pyo3::types::{PyAny, PyBytes, PyDict, PyFloat, PyFrozenSet, PyList, PyTuple}; +use serde::Serialize; +use strum::{IntoStaticStr, VariantArray}; + +static BUNDLED: LazyLock = LazyLock::new(|| { + let loaded = Catalog::parse(BUNDLED_REGISTRY).expect("bundled model registry is valid"); + ParsedRegistry { + catalog: Arc::new(loaded.value), + errors: Arc::from(loaded.errors), + } +}); + +pub(crate) struct ParsedRegistry { + pub(crate) catalog: Arc, + pub(crate) errors: Arc<[Located]>, +} + +#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)] +pub(crate) enum RowErrorKind { + #[strum(serialize = "json")] + Json, + #[strum(serialize = "empty")] + Empty, + #[strum(serialize = "below_minimum")] + BelowMinimum, + #[strum(serialize = "shrunk")] + Shrunk, + #[strum(serialize = "not_an_object")] + NotAnObject, + #[strum(serialize = "invalid")] + Invalid, + #[strum(serialize = "unknown_block_key")] + UnknownBlockKey, + #[strum(serialize = "missing_provider")] + MissingProvider, + #[strum(serialize = "unknown_timezone")] + UnknownTimezone, + #[strum(serialize = "alias_shadows_model")] + AliasShadowsModel, + #[strum(serialize = "alias_taken")] + AliasTaken, + #[strum(serialize = "duplicate_price")] + DuplicatePrice, + #[strum(serialize = "suffix_inside_block")] + SuffixInsideBlock, + #[strum(serialize = "tier_range")] + TierRange, + #[strum(serialize = "tier_reversed")] + TierReversed, + #[strum(serialize = "tier_gap")] + TierGap, + #[strum(serialize = "mixed_tier_basis")] + MixedTierBasis, + #[strum(serialize = "not_priced_by_results")] + NotPricedByResults, + #[strum(serialize = "bands_from_two_sources")] + BandsFromTwoSources, + #[strum(serialize = "not_an_off_peak_rate")] + NotAnOffPeakRate, + #[strum(serialize = "no_off_peak_prices")] + NoOffPeakPrices, + #[strum(serialize = "no_off_peak_hours")] + NoOffPeakHours, + #[strum(serialize = "empty_list")] + EmptyList, + #[strum(serialize = "invalid_date")] + InvalidDate, +} + +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone)] +pub(crate) struct RowError { + #[pyo3(get)] + pub(crate) key: String, + #[pyo3(get)] + pub(crate) field: Option, + #[pyo3(get)] + pub(crate) kind: String, + #[pyo3(get)] + pub(crate) rejects: bool, + #[pyo3(get)] + pub(crate) message: String, +} + +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone)] +pub(crate) struct ModelRegistry { + pub(crate) parsed: Arc, +} + +#[pymethods] +impl ModelRegistry { + #[staticmethod] + fn parse(py: Python<'_>, body: PyBackedBytes) -> PyResult { + let parsed = release_gil(py, move || { + Catalog::parse(&body).map(|loaded| ParsedRegistry { + catalog: Arc::new(loaded.value), + errors: Arc::from(loaded.errors), + }) + }) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(Self { + parsed: Arc::new(parsed), + }) + } + + #[staticmethod] + fn bundled(py: Python<'_>) -> Self { + release_gil(py, || Self { + parsed: Arc::new(ParsedRegistry { + catalog: Arc::clone(&BUNDLED.catalog), + errors: Arc::clone(&BUNDLED.errors), + }), + }) + } + + #[staticmethod] + fn bundled_bytes(py: Python<'_>) -> Bound<'_, PyBytes> { + PyBytes::new(py, BUNDLED_REGISTRY) + } + + fn errors(&self) -> Vec { + self.parsed.errors.iter().map(row_error).collect() + } + + fn to_dict(&self, py: Python<'_>) -> PyResult> { + to_py(py, &WrittenDocument(self.parsed.catalog.as_ref())) + } + + fn served(&self, py: Python<'_>) -> PyResult> { + let catalog = self.parsed.catalog.as_ref(); + let groups = canonical_groups(catalog); + let rows = PyDict::new(py); + let mut converted = HashMap::)>>::new(); + for (key, fields) in catalog.served() { + let canonical = groups.get(key).map_or(key, |canonical| canonical.as_str()); + let fields = fields_without_aliases(fields); + let value = converted.get(canonical).and_then(|projections| { + projections.iter().find_map(|(canonical_fields, row)| { + (canonical_fields == &fields).then(|| row.clone_ref(py)) + }) + }); + let value = match value { + Some(value) => value, + None => { + let value = fields_to_dict(py, &fields)?; + converted + .entry(canonical.to_owned()) + .or_default() + .push((fields, value.clone_ref(py))); + value + } + }; + rows.set_item(key, value)?; + } + Ok(rows.unbind().into_any()) + } + + fn fallback_rules(&self, py: Python<'_>) -> PyResult> { + to_py(py, &self.parsed.catalog.fallback_rules()) + } + + fn check_size(&self, reference: usize) -> Option { + self.parsed + .catalog + .check_size(reference, IntegrityLimits::DEFAULT) + .err() + .as_ref() + .map(row_error) + } + + fn without_rejected(&self) -> (Self, Vec) { + let (catalog, removed) = (*self.parsed.catalog) + .clone() + .without_rejected(&self.parsed.errors); + let removed_errors = removed.into_iter().map(row_error).collect(); + ( + Self { + parsed: Arc::new(ParsedRegistry { + catalog: Arc::new(catalog), + errors: Arc::clone(&self.parsed.errors), + }), + }, + removed_errors, + ) + } +} + +struct WrittenDocument<'a>(&'a Catalog); + +impl Serialize for WrittenDocument<'_> { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_map(self.0.written()) + } +} + +pub(crate) fn row_error(error: &Located) -> RowError { + let mut path = error.path.split('/').skip(1); + let key = unescape_pointer_segment(path.next().unwrap_or_default()); + let field = path.map(unescape_pointer_segment).collect::>(); + let field = (!field.is_empty()).then(|| field.join("/")); + let kind = error_kind(&error.error); + RowError { + key, + field, + kind: <&'static str>::from(kind).to_owned(), + rejects: litellm_model_catalog::rejects_row(&error.error), + message: error.error.to_string(), + } +} + +fn unescape_pointer_segment(segment: &str) -> String { + segment.replace("~1", "/").replace("~0", "~") +} + +fn escape_pointer_segment(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") +} + +fn error_kind(error: &Error) -> RowErrorKind { + match error { + Error::Json(_) => RowErrorKind::Json, + Error::Empty => RowErrorKind::Empty, + Error::BelowMinimum { .. } => RowErrorKind::BelowMinimum, + Error::Shrunk { .. } => RowErrorKind::Shrunk, + Error::NotAnObject => RowErrorKind::NotAnObject, + Error::Invalid(_) => RowErrorKind::Invalid, + Error::UnknownBlockKey => RowErrorKind::UnknownBlockKey, + Error::MissingProvider => RowErrorKind::MissingProvider, + Error::UnknownTimezone { .. } => RowErrorKind::UnknownTimezone, + Error::AliasShadowsModel { .. } => RowErrorKind::AliasShadowsModel, + Error::AliasTaken { .. } => RowErrorKind::AliasTaken, + Error::DuplicatePrice { .. } => RowErrorKind::DuplicatePrice, + Error::SuffixInsideBlock { .. } => RowErrorKind::SuffixInsideBlock, + Error::TierRange => RowErrorKind::TierRange, + Error::TierReversed => RowErrorKind::TierReversed, + Error::TierGap { .. } => RowErrorKind::TierGap, + Error::MixedTierBasis => RowErrorKind::MixedTierBasis, + Error::NotPricedByResults => RowErrorKind::NotPricedByResults, + Error::BandsFromTwoSources => RowErrorKind::BandsFromTwoSources, + Error::NotAnOffPeakRate => RowErrorKind::NotAnOffPeakRate, + Error::NoOffPeakPrices => RowErrorKind::NoOffPeakPrices, + Error::NoOffPeakHours => RowErrorKind::NoOffPeakHours, + Error::EmptyList => RowErrorKind::EmptyList, + Error::InvalidDate => RowErrorKind::InvalidDate, + } +} + +#[pyfunction] +pub(crate) fn model_row_errors(key: &str, row: &Bound<'_, PyAny>) -> PyResult> { + if !row.is_instance_of::() { + return Ok(vec![row_error(&Located::new( + format!("/{}", escape_pointer_segment(key)), + Error::NotAnObject, + ))]); + } + let mut errors = Vec::new(); + if nonfinite_errors( + row, + &format!("/{}", escape_pointer_segment(key)), + &mut errors, + 0, + ) + .is_err() + { + errors.push(row_error(&Located::new( + format!("/{}", escape_pointer_segment(key)), + Error::Invalid(::custom( + "row must contain JSON-compatible values", + )), + ))); + } + if !errors.is_empty() { + return Ok(errors); + } + let fields: Fields = match from_py(row) { + Ok(fields) => fields, + Err(_) => { + let error = ::custom( + "row must contain JSON-compatible values", + ); + return Ok(vec![row_error(&Located::new( + format!("/{}", escape_pointer_segment(key)), + Error::Invalid(error), + ))]); + } + }; + Ok(Catalog::row_errors(key, &fields) + .iter() + .map(row_error) + .collect()) +} + +fn nonfinite_errors( + value: &Bound<'_, PyAny>, + path: &str, + errors: &mut Vec, + depth: usize, +) -> PyResult<()> { + if depth >= 128 { + return Err(PyValueError::new_err("model cost row nesting is too deep")); + } + if let Ok(number) = value.cast::() { + if !number.value().is_finite() { + errors.push(row_error(&Located::new( + path.to_owned(), + Error::Invalid(::custom( + "JSON numbers must be finite", + )), + ))); + } + } else if let Ok(fields) = value.cast::() { + for (key, item) in fields.iter() { + let key: String = key.extract()?; + nonfinite_errors( + &item, + &format!("{path}/{}", escape_pointer_segment(&key)), + errors, + depth + 1, + )?; + } + } else if value.is_instance_of::() || value.is_instance_of::() { + for (index, item) in value.try_iter()?.enumerate() { + nonfinite_errors(&item?, &format!("{path}/{index}"), errors, depth + 1)?; + } + } + Ok(()) +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct ModelCostLayers { + layers: Mutex, +} + +#[pymethods] +impl ModelCostLayers { + #[new] + fn new() -> Self { + Self { + layers: Mutex::default(), + } + } + + fn record_registration( + &self, + py: Python<'_>, + key: String, + inherits_from: Option, + fields: &Bound<'_, PyDict>, + ) -> PyResult<()> { + let fields: SparseRow = from_py(fields)?; + release_gil(py, || { + self.lock().record(ApplicationWrite::Registration { + key, + inherits_from, + fields, + }) + }); + Ok(()) + } + + fn diff(&self, py: Python<'_>, changed: &Bound<'_, PyDict>) -> PyResult<()> { + let changed: BTreeMap, Option)> = from_py(changed)?; + release_gil(py, || { + let mut layers = self.lock(); + for (key, (installed, live)) in changed { + if let Some(write) = diff_row(&key, installed.as_ref(), live.as_ref()) { + layers.record(write); + } + } + }); + Ok(()) + } + + fn forget_writes(&self, py: Python<'_>) { + release_gil(py, || self.lock().forget_writes()); + } + + fn compose(&self, py: Python<'_>, base: &ModelRegistry) -> PyResult> { + let rows = release_gil(py, || { + self.lock() + .compose(&base.parsed.catalog) + .map(|row| (row.key.to_owned(), Some((row.fields, row.registered)))) + .collect() + }); + composed_rows_to_py(py, &base.parsed.catalog, rows) + } + + fn compose_keys( + &self, + py: Python<'_>, + base: &ModelRegistry, + keys: BTreeSet, + ) -> PyResult> { + let rows = release_gil(py, || { + self.lock() + .compose_keys(&base.parsed.catalog, &keys) + .zip(&keys) + .map(|(row, key)| (key.clone(), row.map(|row| (row.fields, row.registered)))) + .collect() + }); + composed_rows_to_py(py, &base.parsed.catalog, rows) + } +} + +impl ModelCostLayers { + fn lock(&self) -> std::sync::MutexGuard<'_, Layers> { + self.layers.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +fn fields_without_aliases(fields: &Fields) -> Fields { + let mut fields = fields.clone(); + fields.shift_remove("aliases"); + fields +} + +fn canonical_groups(catalog: &Catalog) -> HashMap { + catalog + .served() + .map(|(key, _)| { + ( + key.to_owned(), + catalog + .lookup(key) + .map_or(key, |matched| matched.canonical_key) + .to_owned(), + ) + }) + .collect() +} + +fn fields_to_dict(py: Python<'_>, fields: &Fields) -> PyResult> { + let row = PyDict::new(py); + for (key, value) in fields { + row.set_item(key, to_py(py, value)?)?; + } + Ok(row.unbind()) +} + +type ComposedFields = (String, Option<(Fields, BTreeSet)>); +type ConvertedRows = HashMap, Py)>>; + +fn composed_rows_to_py( + py: Python<'_>, + catalog: &Catalog, + rows: Vec, +) -> PyResult> { + let values = PyList::empty(py); + let mut converted = ConvertedRows::new(); + for (key, row) in rows { + let group = catalog + .lookup(&key) + .filter(|matched| matched.matched_key == key) + .map_or(key.as_str(), |matched| matched.canonical_key) + .to_owned(); + let name = key.into_pyobject(py)?.into_any(); + let (fields, registered) = match row { + Some((fields, registered)) => { + let fields = fields_without_aliases(&fields); + let shared = converted.get(&group).and_then(|rows| { + rows.iter().find_map(|(previous, owned, value)| { + (previous == &fields && owned == ®istered).then(|| value.clone_ref(py)) + }) + }); + let value = match shared { + Some(value) => value, + None => { + let value = fields_to_dict(py, &fields)?; + converted.entry(group).or_default().push(( + fields, + registered.clone(), + value.clone_ref(py), + )); + value + } + }; + ( + value.into_bound(py).into_any(), + PyFrozenSet::new(py, registered)?.into_any(), + ) + } + None => (py.None().into_bound(py), PyFrozenSet::empty(py)?.into_any()), + }; + values.append(PyTuple::new(py, [name, fields, registered])?)?; + } + Ok(values.unbind().into_any()) +} + +#[pyfunction] +pub(crate) fn write_rows( + target: &Bound<'_, PyDict>, + set: &Bound<'_, PyDict>, + remove: Vec, +) -> PyResult<()> { + let updates = set + .iter() + .map(|(key, value)| Ok((key.extract::()?, value))) + .collect::>>()?; + // Keep replaced objects alive until the batch is complete, including Python finalizers. + let replaced = updates + .iter() + .map(|(key, _)| key) + .chain(&remove) + .map(|key| target.get_item(key)) + .collect::>>()?; + for (key, value) in updates { + target.set_item(key, value)?; + } + for key in remove { + if target.contains(&key)? { + target.del_item(key)?; + } + } + drop(replaced); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::ffi::c_str; + use rstest::{fixture, rstest}; + use serde_json::{Value, json}; + + #[fixture] + fn scope() -> Py { + Python::initialize(); + Python::attach(|py| { + let native = PyModule::new(py, "native").unwrap(); + native.add_class::().unwrap(); + native.add_class::().unwrap(); + native.add_class::().unwrap(); + native + .add_function(wrap_pyfunction!(model_row_errors, &native).unwrap()) + .unwrap(); + native + .add_function(wrap_pyfunction!(write_rows, &native).unwrap()) + .unwrap(); + let scope = PyDict::new(py); + scope.set_item("native", native).unwrap(); + scope.unbind() + }) + } + + fn json_error() -> serde_json::Error { + serde_json::from_str::("{").unwrap_err() + } + + #[rstest] + #[case::json(Error::Json(json_error()), "json", false)] + #[case::empty(Error::Empty, "empty", false)] + #[case::minimum(Error::BelowMinimum { actual: 1, minimum: 100 }, "below_minimum", false)] + #[case::shrunk(Error::Shrunk { actual: 100, reference: 300, ratio: 0.5 }, "shrunk", false)] + #[case::object(Error::NotAnObject, "not_an_object", true)] + #[case::invalid(Error::Invalid(json_error()), "invalid", true)] + #[case::block(Error::UnknownBlockKey, "unknown_block_key", true)] + #[case::provider(Error::MissingProvider, "missing_provider", false)] + #[case::timezone(Error::UnknownTimezone { zone: "bad".into() }, "unknown_timezone", true)] + #[case::shadow(Error::AliasShadowsModel { alias: "m".into() }, "alias_shadows_model", true)] + #[case::taken(Error::AliasTaken { alias: "a".into(), taken_by: "m".into() }, "alias_taken", true)] + #[case::duplicate(Error::DuplicatePrice { first: "a".into(), second: "b".into() }, "duplicate_price", true)] + #[case::suffix(Error::SuffixInsideBlock { block: "tiered_pricing" }, "suffix_inside_block", true)] + #[case::range(Error::TierRange, "tier_range", true)] + #[case::reversed(Error::TierReversed, "tier_reversed", true)] + #[case::gap(Error::TierGap { start: 3, expected: 2 }, "tier_gap", true)] + #[case::basis(Error::MixedTierBasis, "mixed_tier_basis", true)] + #[case::results(Error::NotPricedByResults, "not_priced_by_results", true)] + #[case::bands(Error::BandsFromTwoSources, "bands_from_two_sources", true)] + #[case::rate(Error::NotAnOffPeakRate, "not_an_off_peak_rate", true)] + #[case::prices(Error::NoOffPeakPrices, "no_off_peak_prices", true)] + #[case::hours(Error::NoOffPeakHours, "no_off_peak_hours", true)] + #[case::list(Error::EmptyList, "empty_list", true)] + #[case::date(Error::InvalidDate, "invalid_date", true)] + fn row_error_matches_catalog_policy( + scope: Py, + #[case] error: Error, + #[case] kind: &str, + #[case] rejects: bool, + ) { + assert_eq!(litellm_model_catalog::rejects_row(&error), rejects); + let message = error.to_string(); + Python::attach(|py| { + let scope = scope.bind(py); + scope + .set_item( + "error", + row_error(&Located::new("/team~1model~0/tiers/0/input", error)), + ) + .unwrap(); + scope.set_item("kind", kind).unwrap(); + scope.set_item("rejects", rejects).unwrap(); + scope.set_item("message", message).unwrap(); + py.run( + c_str!( + r#" +assert (error.key, error.field, error.kind, error.rejects, error.message) == ( + "team/model~", "tiers/0/input", kind, rejects, message) +try: + error.kind = "invalid" +except AttributeError: + pass +else: + raise AssertionError("row errors must be immutable") +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + fn registry_preserves_written_sections_and_shares_served_aliases(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run( + c_str!( + r#" +import json +document = { + "sample_spec": {"description": "not a model"}, + "model": {"litellm_provider": "openai", "aliases": ["alias"], "input_cost_per_token": 0}, + "bad": {"litellm_provider": "openai", "input_cost_per_token": "invalid"}, + "no-provider": {"input_cost_per_token": 1}, + "non-object": 42, +} +registry = native.ModelRegistry.parse(json.dumps(document).encode()) +assert registry.to_dict() == document +assert registry.fallback_rules() is None +served = registry.served() +assert set(served) == {"model", "alias", "bad", "no-provider"} +assert served["model"] is served["alias"] +assert served["model"] == {"litellm_provider": "openai", "input_cost_per_token": 0} +assert {(e.key, e.kind, e.rejects) for e in registry.errors()} == { + ("bad", "invalid", True), ("no-provider", "missing_provider", False), + ("non-object", "not_an_object", True), +} +clean, rejected = registry.without_rejected() +assert {e.key for e in rejected} == {"bad", "non-object"} +assert set(clean.served()) == {"model", "alias", "no-provider"} +assert "bad" in registry.to_dict() +assert registry.check_size(0).kind == "below_minimum" +assert registry.check_size(0).key == "" +assert registry.check_size(0).field is None +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + #[case::json(b"{")] + #[case::root(b"[]")] + #[case::empty(b"{}")] + fn fatal_parse_errors_are_value_errors(scope: Py, #[case] body: &[u8]) { + Python::attach(|py| { + let scope = scope.bind(py); + scope.set_item("body", PyBytes::new(py, body)).unwrap(); + py.run( + c_str!( + r#" +try: + native.ModelRegistry.parse(body) +except ValueError as error: + assert str(error) +else: + raise AssertionError("expected fatal catalog parse error") +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + fn bundled_bytes_and_registry_agree(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run( + c_str!( + r#" +import json +bundled = native.ModelRegistry.bundled() +body = native.ModelRegistry.bundled_bytes() +assert isinstance(body, bytes) +assert bundled.to_dict() == json.loads(body) +assert bundled.served() == native.ModelRegistry.parse(body).served() +assert bundled.fallback_rules() == bundled.to_dict()["fallback_generalizations"]["rules"] +assert bundled.check_size(0) is None +assert bundled.check_size(len(bundled.served()) * 10).kind == "shrunk" +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + #[case::nan(f64::NAN)] + #[case::positive_infinity(f64::INFINITY)] + #[case::negative_infinity(f64::NEG_INFINITY)] + fn nonfinite_values_keep_their_nested_paths(scope: Py, #[case] number: f64) { + Python::attach(|py| { + let scope = scope.bind(py); + scope.set_item("number", number).unwrap(); + py.run( + c_str!( + r#" +errors = native.model_row_errors("team/model~", { + "litellm_provider": "openai", + "tiered_pricing": [{"input_cost_per_token": number}], +}) +assert [(e.key, e.field, e.kind, e.rejects) for e in errors] == [ + ("team/model~", "tiered_pricing/0/input_cost_per_token", "invalid", True)] +assert "finite" in errors[0].message +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + fn row_validation_keeps_missing_provider_nonrejecting(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run(c_str!(r#" +assert native.model_row_errors("model", {"litellm_provider": "openai", "input_cost_per_token": 0}) == [] +errors = native.model_row_errors("team/model~", {"input_cost_per_token": 0}) +assert [(e.key, e.field, e.kind, e.rejects) for e in errors] == [ + ("team/model~", None, "missing_provider", False)] +cyclic = {} +cyclic["self"] = cyclic +errors = native.model_row_errors("model", cyclic) +assert [(e.key, e.field, e.kind, e.rejects) for e in errors] == [("model", None, "invalid", True)] +assert "JSON-compatible" in errors[0].message +"#), Some(scope), Some(scope)).unwrap(); + }); + } + + #[rstest] + #[case::absent(None, None, Some(json!({"litellm_provider":"openai", "input_cost_per_token":2, "cache_read_input_token_cost":4})))] + #[case::insert(None, Some(json!({"input_cost_per_token":7})), Some(json!({"litellm_provider":"openai", "input_cost_per_token":7, "cache_read_input_token_cost":4})))] + #[case::delete(Some(json!({"input_cost_per_token":1})), None, None)] + #[case::edit(Some(json!({"input_cost_per_token":1, "cache_read_input_token_cost":3})), Some(json!({"input_cost_per_token":7})), Some(json!({"litellm_provider":"openai", "input_cost_per_token":7})))] + #[case::equal(Some(json!({"input_cost_per_token":1})), Some(json!({"input_cost_per_token":1})), Some(json!({"litellm_provider":"openai", "input_cost_per_token":2, "cache_read_input_token_cost":4})))] + #[case::nested(Some(json!({"metadata":{"keep":1,"remove":2}})), Some(json!({"metadata":{"keep":1}})), Some(json!({"litellm_provider":"openai", "input_cost_per_token":2, "cache_read_input_token_cost":4,"metadata":{"keep":1}})))] + fn diff_composes_sparse_changes_onto_new_base( + scope: Py, + #[case] installed: Option, + #[case] live: Option, + #[case] expected: Option, + ) { + Python::attach(|py| { + let scope = scope.bind(py); + scope + .set_item("installed", to_py(py, &installed).unwrap()) + .unwrap(); + scope.set_item("live", to_py(py, &live).unwrap()).unwrap(); + scope + .set_item("expected", to_py(py, &expected).unwrap()) + .unwrap(); + py.run(c_str!(r#" +base = native.ModelRegistry.parse(b'{"model":{"litellm_provider":"openai","input_cost_per_token":2,"cache_read_input_token_cost":4}}') +layers = native.ModelCostLayers() +layers.diff({"model": (installed, live)}) +assert layers.compose_keys(base, frozenset({"model", "missing"})) == [ + ("missing", None, frozenset()), ("model", expected, frozenset())] +assert layers.compose(base) == ([] if expected is None else [("model", expected, frozenset())]) +layers.forget_writes() +assert layers.compose(base) == [("model", base.served()["model"], frozenset())] +"#), Some(scope), Some(scope)).unwrap(); + }); + } + + #[rstest] + fn registrations_keep_inheritance_and_direct_ownership_separate(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run(c_str!(r#" +base = native.ModelRegistry.parse(b'{"model":{"litellm_provider":"openai","input_cost_per_token":2,"cache_read_input_token_cost":4}}') +layers = native.ModelCostLayers() +layers.record_registration("openai/model", "model", {"input_cost_per_token":5, "max_tokens":100}) +registered = {"litellm_provider":"openai", "input_cost_per_token":5, "cache_read_input_token_cost":4, "max_tokens":100} +assert layers.compose_keys(base, frozenset({"openai/model"})) == [ + ("openai/model", registered, frozenset({"input_cost_per_token", "max_tokens"}))] +live = dict(registered, input_cost_per_token=7) +del live["cache_read_input_token_cost"] +layers.diff({"openai/model": (registered, live), "z": (None, {"extra":None}), "a": (None, {})}) +assert layers.compose(base) == [ + ("model", base.served()["model"], frozenset()), + ("openai/model", live, frozenset({"max_tokens"})), + ("a", {}, frozenset()), ("z", {"extra":None}, frozenset()), +] +layers.record_registration("openai/model", "model", {"input_cost_per_token":9}) +assert layers.compose_keys(base, frozenset({"openai/model"})) == [ + ("openai/model", dict(live, input_cost_per_token=9), frozenset({"max_tokens", "input_cost_per_token"}))] +layers.diff({"openai/model": (live, None)}) +assert layers.compose_keys(base, frozenset({"openai/model"})) == [("openai/model", None, frozenset())] +"#), Some(scope), Some(scope)).unwrap(); + }); + } + + #[rstest] + fn composition_shares_only_unchanged_alias_projections(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run(c_str!(r#" +base = native.ModelRegistry.parse(b'{"model":{"litellm_provider":"openai","aliases":["alias"],"input_cost_per_token":2}}') +layers = native.ModelCostLayers() +initial = {key: row for key,row,_ in layers.compose(base)} +assert initial["model"] is initial["alias"] +layers.diff({"MODEL": (None, initial["model"])}) +distinct = {key: row for key,row,_ in layers.compose(base)} +assert distinct["MODEL"] == distinct["model"] +assert distinct["MODEL"] is not distinct["model"] +layers.record_registration("alias", None, {"input_cost_per_token":2}) +registered = {key: row for key,row,_ in layers.compose(base)} +assert registered["model"] == registered["alias"] +assert registered["model"] is not registered["alias"] +layers.diff({"alias": (initial["alias"], dict(initial["alias"], input_cost_per_token=3))}) +assert layers.compose_keys(base, frozenset({"model", "alias"})) == [ + ("alias", {"litellm_provider":"openai", "input_cost_per_token":3}, frozenset()), + ("model", initial["model"], frozenset()), +] +"#), Some(scope), Some(scope)).unwrap(); + }); + } + + #[rstest] + fn write_rows_bypasses_subclass_hooks_and_delays_finalizers(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run( + c_str!( + r#" +class LiveDict(dict): + def __setitem__(self, key, value): + raise AssertionError("Python set hook called") + def __delitem__(self, key): + raise AssertionError("Python delete hook called") + def clear(self): + raise AssertionError("clear called") +observed = [] +class Row(dict): + def __del__(self): + observed.append(dict(target)) +target = LiveDict({"replaced": Row(), "removed": Row(), "untouched": {}}) +untouched = target["untouched"] +new = {"replaced": {"rate":0}, "new": {}} +assert native.write_rows(target, new, ["removed", "missing"]) is None +expected = dict(new, untouched={}) +assert type(target) is LiveDict +assert target == expected +assert target["untouched"] is untouched +assert target["replaced"] is new["replaced"] +assert observed == [expected, expected] +native.write_rows(target, target, ["new"]) +assert target == {"untouched":{}, "replaced":{"rate":0}} +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + fn write_rows_is_atomic_to_python_observers(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run( + c_str!( + r#" +import sys +import threading +target = {f"old-{i}":{} for i in range(50)} +updates = {f"new-{i}":{} for i in range(3999)} | {"new-key":{}} +remove = list(target) +observed_sizes = set() +observed_membership = set() +ready, finished, after = threading.Event(), threading.Event(), threading.Event() +def observe(): + while not finished.is_set(): + observed_sizes.add(len(target)) + observed_membership.add("new-key" in target) + ready.set() + if "new-key" in target: + after.set() +thread = threading.Thread(target=observe, daemon=True) +interval = sys.getswitchinterval() +try: + sys.setswitchinterval(0.000001) + thread.start() + assert ready.wait(5) + native.write_rows(target, updates, remove) + assert after.wait(5) +finally: + finished.set() + thread.join(5) + sys.setswitchinterval(interval) +assert not thread.is_alive() +assert target == updates +assert observed_sizes == {50, 4000}, observed_sizes +assert observed_membership == {False, True} +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } + + #[rstest] + fn compose_releases_gil_for_a_python_thread(scope: Py) { + Python::attach(|py| { + let scope = scope.bind(py); + py.run( + c_str!( + r#" +import sys +import threading +base = native.ModelRegistry.bundled() +layers = native.ModelCostLayers() +ready, begin, ran = threading.Event(), threading.Event(), threading.Event() +def observer(): + ready.set() + begin.wait() + ran.set() +thread = threading.Thread(target=observer, daemon=True) +thread.start() +assert ready.wait(5) +interval = sys.getswitchinterval() +try: + sys.setswitchinterval(60) + begin.set() + rows = layers.compose(base) + ran_during_compose = ran.is_set() +finally: + sys.setswitchinterval(interval) + thread.join(5) +assert ran_during_compose, "compose held the GIL" +assert {key:row for key,row,_ in rows} == base.served() +"# + ), + Some(scope), + Some(scope), + ) + .unwrap(); + }); + } +} diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 7364a1d72f2..34926684c55 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,6 +1,6 @@ from asyncio import Future from collections.abc import Mapping, Sequence -from typing import Never, final +from typing import Literal, Never, final import httpx from pydantic import JsonValue @@ -13,6 +13,75 @@ class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... +RowErrorKind = Literal[ + "json", + "empty", + "below_minimum", + "shrunk", + "not_an_object", + "invalid", + "unknown_block_key", + "missing_provider", + "unknown_timezone", + "alias_shadows_model", + "alias_taken", + "duplicate_price", + "suffix_inside_block", + "tier_range", + "tier_reversed", + "tier_gap", + "mixed_tier_basis", + "not_priced_by_results", + "bands_from_two_sources", + "not_an_off_peak_rate", + "no_off_peak_prices", + "no_off_peak_hours", + "empty_list", + "invalid_date", +] + +@final +class RowError: + @property + def key(self) -> str: ... + @property + def field(self) -> str | None: ... + @property + def kind(self) -> RowErrorKind: ... + @property + def rejects(self) -> bool: ... + @property + def message(self) -> str: ... + +@final +class ModelRegistry: + @staticmethod + def parse(body: bytes) -> ModelRegistry: ... + @staticmethod + def bundled() -> ModelRegistry: ... + @staticmethod + def bundled_bytes() -> bytes: ... + def errors(self) -> list[RowError]: ... + def to_dict(self) -> dict[str, JsonValue]: ... + def served(self) -> dict[str, dict[str, JsonValue]]: ... + def fallback_rules(self) -> list[dict[str, JsonValue]] | None: ... + def without_rejected(self) -> tuple[ModelRegistry, list[RowError]]: ... + def check_size(self, reference: int) -> RowError | None: ... + +def model_row_errors(key: str, row: dict[str, JsonValue]) -> list[RowError]: ... +def write_rows(target: dict[str, object], set: dict[str, object], remove: list[str]) -> None: ... + +@final +class ModelCostLayers: + def __new__(cls) -> ModelCostLayers: ... + def record_registration(self, key: str, inherits_from: str | None, fields: dict[str, JsonValue]) -> None: ... + def diff(self, changed: dict[str, tuple[dict[str, JsonValue] | None, dict[str, JsonValue] | None]]) -> None: ... + def forget_writes(self) -> None: ... + def compose(self, base: ModelRegistry) -> list[tuple[str, dict[str, JsonValue], frozenset[str]]]: ... + def compose_keys( + self, base: ModelRegistry, keys: frozenset[str] + ) -> list[tuple[str, dict[str, JsonValue] | None, frozenset[str]]]: ... + def trace_encode_error(message: str) -> bytes: ... @final @@ -205,11 +274,15 @@ def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", + "ModelCostLayers", + "ModelRegistry", "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeDiagnosticProcessor", "ProcessReservedForForking", "ResponsesWebSocketConnection", + "RowError", + "RowErrorKind", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", @@ -224,6 +297,7 @@ __all__ = [ "embedding", "gil_stats", "messages", + "model_row_errors", "ocr", "ocr_health_check_document", "ocr_passthrough_response", @@ -232,6 +306,7 @@ __all__ = [ "responses", "trace_encode_error", "transcription", + "write_rows", ] @final