diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 775bd490060..3c1844c41a1 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -564,51 +564,4 @@ mod tests { .any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules) ); } - use serde_json::json; - - fn parse(document: Value) -> Loaded { - Catalog::parse(document.to_string().as_bytes()).unwrap() - } - - #[rstest] - fn a_rejected_row_is_replaced_by_bundled_or_left_out() { - let bundled = - parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}})) - .value; - let fetched = parse(json!({ - "gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1}, - "acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"}, - "kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0} - })); - - let (catalog, rejections) = fetched - .value - .with_rejected_replaced(&fetched.errors, &bundled); - - let replaced: Vec<(&str, bool)> = rejections - .iter() - .map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled)) - .collect(); - assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]); - assert_eq!( - catalog - .lookup("gpt-4o") - .unwrap() - .entry - .field("input_cost_per_token"), - Some(&json!(1.0)) - ); - assert_eq!(catalog.lookup("GPT-4O").unwrap().matched_key, "gpt-4o"); - assert!(catalog.lookup("acme/new").is_none()); - assert!(catalog.lookup("ACME/NEW").is_none()); - assert!(catalog.lookup("kept").is_some()); - } - - #[rstest] - fn a_null_row_is_a_removal_not_an_error() { - let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null})); - assert!(loaded.errors.is_empty(), "{:?}", loaded.errors); - assert_eq!(loaded.value.removals().collect::>(), ["retired"]); - assert_eq!(loaded.value.served().count(), 1); - } } diff --git a/litellm-rust/crates/model-catalog/src/install.rs b/litellm-rust/crates/model-catalog/src/install.rs new file mode 100644 index 00000000000..6a325387e80 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/install.rs @@ -0,0 +1,271 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use indexmap::IndexMap; +use serde_json::Value; + +use crate::catalog::Catalog; +use crate::layers::Layers; +use crate::wire::Fields; + +/// What the bound dict holds for one field, as litellm compares it +#[derive(Clone, Debug, PartialEq)] +pub enum Held { + Absent, + Json(Value), + /// A value that does not convert to JSON, which only the user can have written + Foreign, +} + +/// What the bound dict holds under one key +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Slot { + Missing, + /// Anything but a dict, which only the user can have written + NotARow, + Row, +} + +/// The dict install writes into, `litellm.model_cost` in the Python host +pub trait BoundMap { + type Error; + + fn slot(&self, key: &str) -> Result; + /// Called only for a key whose slot is `Row` + fn held(&self, key: &str, field: &str) -> Result; + /// The row's string-named fields, called only for a key whose slot is `Row` + fn fields(&self, key: &str) -> Result, Self::Error>; + fn row_is_empty(&self, key: &str) -> Result; + fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Self::Error>; + fn del_field(&mut self, key: &str, field: &str) -> Result<(), Self::Error>; + /// A new row under `key`, the same object for every key given the same `group` in one install + fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Self::Error>; + fn del_key(&mut self, key: &str) -> Result<(), Self::Error>; + /// Deletes every key, string or not, that `keep` refuses, and returns how many + fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result; +} + +/// The field values litellm last wrote into one dict object +#[derive(Clone, Debug)] +pub struct Record { + dict_id: usize, + rows: HashMap, +} + +#[derive(Debug)] +pub struct Outcome { + pub record: Record, + /// Keys set because the row differed or was new + pub written: usize, + /// Keys deleted because the map no longer serves them + pub removed: usize, +} + +/// The layers and what the last install wrote, kept between installs +#[derive(Debug)] +pub struct Installs { + layers: Layers, + record: Option, +} + +/// One install between `Installs::begin` and `Installs::finish` +#[derive(Debug)] +pub struct Pending { + layers: Layers, + previous: Option, + dict_id: usize, +} + +impl Installs { + pub fn new(bundled: Arc) -> Self { + Self { + layers: Layers::new(bundled), + record: None, + } + } + + /// Replaces the fetched layer. The last record applies only while `dict_id` names the dict it was written into + pub fn begin(&mut self, fetched: Arc, dict_id: usize) -> Pending { + self.layers.replace_fetched(fetched); + Pending { + layers: self.layers.clone(), + previous: self + .record + .clone() + .filter(|record| record.dict_id == dict_id), + dict_id, + } + } + + pub fn finish(&mut self, record: Record) { + self.record = Some(record); + } +} + +impl Pending { + pub fn compose(&self) -> IndexMap { + self.layers.compose() + } + + /// A field is written only while litellm still owns it, meaning the dict holds the value last written there, or + /// nothing where nothing was written. Without a record for this dict the dict is written in full, which is main's + /// result after rebind then adopt + pub fn write( + self, + map: &mut M, + composed: &IndexMap, + ) -> Result { + match self.previous { + Some(previous) => write_owned(map, composed, &self.layers, previous), + None => write_in_full(map, composed, &self.layers, self.dict_id), + } + } +} + +fn write_owned( + map: &mut M, + composed: &IndexMap, + layers: &Layers, + previous: Record, +) -> Result { + let mut rows = previous.rows; + let mut written = 0; + let mut removed = 0; + let keys: Vec = composed + .keys() + .chain(rows.keys().filter(|key| !composed.contains_key(*key))) + .cloned() + .collect(); + for key in keys { + let target = composed.get(&key); + let last = rows.remove(&key); + match map.slot(&key)? { + Slot::Missing => match (target, last) { + (Some(target), None) => { + map.insert_row(&key, group(layers, &key), target)?; + rows.insert(key, target.clone()); + written += 1; + } + (Some(_), Some(last)) => { + rows.insert(key, last); + } + (None, _) => {} + }, + Slot::NotARow => { + if let Some(last) = last { + rows.insert(key, last); + } + } + Slot::Row => { + let last = last.unwrap_or_default(); + let (owned, changed) = write_owned_fields(map, &key, target, &last)?; + if target.is_none() && changed && map.row_is_empty(&key)? { + map.del_key(&key)?; + removed += 1; + continue; + } + written += usize::from(changed); + if !owned.is_empty() { + rows.insert(key, owned); + } + } + } + } + Ok(Outcome { + record: Record { + dict_id: previous.dict_id, + rows, + }, + written, + removed, + }) +} + +fn write_in_full( + map: &mut M, + composed: &IndexMap, + layers: &Layers, + dict_id: usize, +) -> Result { + let removed = map.retain_keys(&|key| composed.contains_key(key))?; + let mut written = 0; + for (key, target) in composed { + if map.slot(key)? != Slot::Row { + map.insert_row(key, group(layers, key), target)?; + written += 1; + continue; + } + let mut current = Fields::new(); + for (field, held) in map.fields(key)? { + match held { + Held::Json(value) => { + current.insert(field, value); + } + Held::Absent | Held::Foreign => map.del_field(key, &field)?, + } + } + let (_, changed) = write_owned_fields(map, key, Some(target), ¤t)?; + written += usize::from(changed); + } + Ok(Outcome { + record: Record { + dict_id, + rows: composed + .iter() + .map(|(key, fields)| (key.clone(), fields.clone())) + .collect(), + }, + written, + removed, + }) +} + +/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed +fn write_owned_fields( + map: &mut M, + key: &str, + target: Option<&Fields>, + last: &Fields, +) -> Result<(Fields, bool), M::Error> { + let empty = Fields::new(); + let target = target.unwrap_or(&empty); + let fields: Vec<&String> = target + .keys() + .chain(last.keys().filter(|field| !target.contains_key(*field))) + .collect(); + let mut owned = Fields::new(); + let mut changed = false; + for field in fields { + let held = map.held(key, field)?; + let litellm_owns = match (&held, last.get(field)) { + (Held::Absent, None) => true, + (Held::Json(value), Some(last)) => value == last, + _ => false, + }; + if !litellm_owns { + if let Some(last) = last.get(field) { + owned.insert(field.clone(), last.clone()); + } + continue; + } + match target.get(field) { + Some(value) => { + if held != Held::Json(value.clone()) { + map.set_field(key, field, value)?; + changed = true; + } + owned.insert(field.clone(), value.clone()); + } + None if held != Held::Absent => { + map.del_field(key, field)?; + changed = true; + } + None => {} + } + } + Ok((owned, changed)) +} + +fn group<'a>(layers: &'a Layers, key: &'a str) -> &'a str { + layers.alias_owner(key).unwrap_or(key) +} diff --git a/litellm-rust/crates/model-catalog/src/layers.rs b/litellm-rust/crates/model-catalog/src/layers.rs index c39647e3f5a..6b0ae8be17d 100644 --- a/litellm-rust/crates/model-catalog/src/layers.rs +++ b/litellm-rust/crates/model-catalog/src/layers.rs @@ -101,63 +101,3 @@ fn without_aliases(fields: &Fields) -> Fields { .map(|(field, value)| (field.clone(), value.clone())) .collect() } - -#[cfg(test)] -mod tests { - use rstest::rstest; - use serde_json::json; - - use super::*; - - fn catalog(document: Value) -> Arc { - Arc::new( - Catalog::parse(document.to_string().as_bytes()) - .unwrap() - .value, - ) - } - - fn bundled() -> Arc { - catalog(json!({ - "gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384}, - "retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0} - })) - } - - fn composed(fetched: Value) -> IndexMap { - let mut layers = Layers::new(bundled()); - layers.replace_fetched(catalog(fetched)); - layers.compose() - } - - #[rstest] - fn a_field_the_fetched_row_lacks_is_served_from_bundled() { - let rows = composed( - json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}), - ); - assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0)); - assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384)); - } - - #[rstest] - fn a_null_field_removes_it() { - let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}})); - assert!(!rows["gpt-4o"].contains_key("max_tokens")); - } - - #[rstest] - fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() { - let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}})); - let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null})); - assert!(kept.contains_key("retired")); - assert!(!retired.contains_key("retired")); - } - - #[rstest] - fn aliases_are_served_under_every_name_without_the_aliases_field() { - let rows = - composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}})); - assert_eq!(rows["new-alias"], rows["new"]); - assert!(!rows["new"].contains_key("aliases")); - } -} diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs index fe419dde054..a2b704bd7a5 100644 --- a/litellm-rust/crates/model-catalog/src/lib.rs +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -39,6 +39,7 @@ mod bundled; mod catalog; mod error; mod fallback; +mod install; mod layers; mod lower; mod model; @@ -53,6 +54,7 @@ pub use bundled::*; pub use catalog::*; pub use error::*; pub use fallback::*; +pub use install::*; pub use layers::*; pub use model::*; pub use model_info::*; diff --git a/litellm-rust/crates/model-catalog/tests/policy.rs b/litellm-rust/crates/model-catalog/tests/policy.rs new file mode 100644 index 00000000000..9d7169b854d --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/policy.rs @@ -0,0 +1,511 @@ +use std::collections::HashMap; +use std::convert::Infallible; +use std::sync::Arc; + +use indexmap::IndexMap; +use litellm_model_catalog::wire::Fields; +use litellm_model_catalog::{ + BoundMap, Catalog, Error, Held, Installs, IntegrityLimits, Outcome, Pending, Slot, +}; +use rstest::rstest; +use serde_json::{Value, json}; + +const FOREIGN: &str = ""; +const NOT_A_ROW: &str = ""; + +fn bundled_document() -> Value { + json!({ + "sample_spec": {"litellm_provider": "one of the providers", "input_cost_per_token": 0.0}, + "gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384}, + "retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0}, + "claude": {"litellm_provider": "anthropic", "input_cost_per_token": 3.0, "aliases": ["claude-latest"]} + }) +} + +fn parse(document: &Value) -> Catalog { + Catalog::parse(document.to_string().as_bytes()) + .unwrap() + .value +} + +/// What python-bridge's `parse` hands to install for a fetched body +fn fetched(document: &Value, bundled: &Catalog) -> Arc { + let loaded = Catalog::parse(document.to_string().as_bytes()).unwrap(); + Arc::new( + loaded + .value + .with_rejected_replaced(&loaded.errors, bundled) + .0, + ) +} + +#[derive(Clone, Debug, PartialEq)] +enum Stored { + Json(Value), + Foreign, +} + +#[derive(Clone, Debug)] +enum Entry { + Row(usize), + NotARow, +} + +/// A Python dict of rows, where two keys can hold the same row object +#[derive(Clone, Debug, Default)] +struct Dict { + keys: IndexMap, + rows: Vec>, +} + +struct Session<'a> { + dict: &'a mut Dict, + groups: HashMap, +} + +impl Dict { + fn row_mut(&mut self, key: &str) -> &mut IndexMap { + match self.keys[key] { + Entry::Row(index) => &mut self.rows[index], + Entry::NotARow => panic!("{key} is not a row"), + } + } + + fn view(&self, key: &str) -> Value { + match self.keys.get(key) { + None => Value::Null, + Some(Entry::NotARow) => json!(NOT_A_ROW), + Some(Entry::Row(index)) => self.rows[*index] + .iter() + .map(|(field, stored)| { + let value = match stored { + Stored::Json(value) => value.clone(), + Stored::Foreign => json!(FOREIGN), + }; + (field.clone(), value) + }) + .collect(), + } + } + + fn shares_row(&self, a: &str, b: &str) -> bool { + matches!( + (&self.keys[a], &self.keys[b]), + (Entry::Row(a), Entry::Row(b)) if a == b + ) + } +} + +impl BoundMap for Session<'_> { + type Error = Infallible; + + fn slot(&self, key: &str) -> Result { + Ok(match self.dict.keys.get(key) { + None => Slot::Missing, + Some(Entry::NotARow) => Slot::NotARow, + Some(Entry::Row(_)) => Slot::Row, + }) + } + + fn held(&self, key: &str, field: &str) -> Result { + let Entry::Row(index) = self.dict.keys[key] else { + panic!("{key} is not a row"); + }; + Ok(match self.dict.rows[index].get(field) { + None => Held::Absent, + Some(Stored::Json(value)) => Held::Json(value.clone()), + Some(Stored::Foreign) => Held::Foreign, + }) + } + + fn fields(&self, key: &str) -> Result, Infallible> { + let Entry::Row(index) = self.dict.keys[key] else { + panic!("{key} is not a row"); + }; + Ok(self.dict.rows[index] + .iter() + .map(|(field, stored)| { + let held = match stored { + Stored::Json(value) => Held::Json(value.clone()), + Stored::Foreign => Held::Foreign, + }; + (field.clone(), held) + }) + .collect()) + } + + fn row_is_empty(&self, key: &str) -> Result { + Ok(matches!(self.dict.keys[key], Entry::Row(index) if self.dict.rows[index].is_empty())) + } + + fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Infallible> { + self.dict + .row_mut(key) + .insert(field.to_owned(), Stored::Json(value.clone())); + Ok(()) + } + + fn del_field(&mut self, key: &str, field: &str) -> Result<(), Infallible> { + self.dict.row_mut(key).shift_remove(field); + Ok(()) + } + + fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Infallible> { + let index = match self.groups.get(group) { + Some(index) => *index, + None => { + self.dict.rows.push( + fields + .iter() + .map(|(field, value)| (field.clone(), Stored::Json(value.clone()))) + .collect(), + ); + let index = self.dict.rows.len() - 1; + self.groups.insert(group.to_owned(), index); + index + } + }; + self.dict.keys.insert(key.to_owned(), Entry::Row(index)); + Ok(()) + } + + fn del_key(&mut self, key: &str) -> Result<(), Infallible> { + self.dict.keys.shift_remove(key); + Ok(()) + } + + fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result { + let before = self.dict.keys.len(); + self.dict.keys.retain(|key, _| keep(key)); + Ok(before - self.dict.keys.len()) + } +} + +/// What the user does to `litellm.model_cost` between two installs +#[derive(Clone, Debug)] +enum Edit { + Set(&'static str, &'static str, Value), + Unset(&'static str, &'static str), + SetForeign(&'static str, &'static str), + Pop(&'static str), + Insert(&'static str, Value), + ReplaceWithNonRow(&'static str), + /// `litellm.model_cost = dict(litellm.model_cost)`, the copy landing at `dict_id` + RebindToCopy { + dict_id: usize, + }, +} + +/// One Python process: the bundled map installed at import, then the user and later installs +struct World { + bundled: Arc, + installs: Installs, + dict: Dict, + dict_id: usize, +} + +impl World { + fn new() -> Self { + let bundled = Arc::new(parse(&bundled_document())); + let mut world = Self { + installs: Installs::new(Arc::clone(&bundled)), + bundled: Arc::clone(&bundled), + dict: Dict::default(), + dict_id: 1, + }; + world.install_catalog(bundled); + world + } + + fn begin(&mut self, document: &Value) -> Pending { + let catalog = fetched(document, &self.bundled); + self.installs.begin(catalog, self.dict_id) + } + + fn write(&mut self, pending: Pending) -> Outcome { + let composed = pending.compose(); + let mut session = Session { + dict: &mut self.dict, + groups: HashMap::new(), + }; + let Ok(outcome) = pending.write(&mut session, &composed); + outcome + } + + fn install_catalog(&mut self, catalog: Arc) { + let pending = self.installs.begin(catalog, self.dict_id); + let outcome = self.write(pending); + self.installs.finish(outcome.record); + } + + fn install(&mut self, document: &Value) { + let catalog = fetched(document, &self.bundled); + self.install_catalog(catalog); + } + + fn edit(&mut self, edit: &Edit) { + match edit { + Edit::Set(key, field, value) => { + self.dict + .row_mut(key) + .insert((*field).to_owned(), Stored::Json(value.clone())); + } + Edit::Unset(key, field) => { + self.dict.row_mut(key).shift_remove(*field); + } + Edit::SetForeign(key, field) => { + self.dict + .row_mut(key) + .insert((*field).to_owned(), Stored::Foreign); + } + Edit::Pop(key) => { + self.dict.keys.shift_remove(*key); + } + Edit::Insert(key, row) => { + self.dict.rows.push( + row.as_object() + .unwrap() + .iter() + .map(|(field, value)| (field.clone(), Stored::Json(value.clone()))) + .collect(), + ); + let index = self.dict.rows.len() - 1; + self.dict.keys.insert((*key).to_owned(), Entry::Row(index)); + } + Edit::ReplaceWithNonRow(key) => { + self.dict.keys.insert((*key).to_owned(), Entry::NotARow); + } + Edit::RebindToCopy { dict_id } => { + self.dict = self.dict.clone(); + self.dict_id = *dict_id; + } + } + } + + fn row(&self, key: &str) -> Value { + self.dict.view(key) + } +} + +fn gpt_4o(price: f64) -> Value { + json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price}}) +} + +fn gpt_4o_row(price: f64, max_tokens: u32) -> Value { + json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price, "max_tokens": max_tokens}}) +} + +/// The behavior users expect. A fetched map replaces bundled at row level, bundled is only the fallback for a bad +/// row or a bad document, and the user's own writes survive every install. Cases the code does not meet yet fail +mod expected { + use super::*; + + #[rstest] + #[case::fetched_row_is_served_as_written( + gpt_4o(2.0), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 2.0}) + )] + #[case::null_field_is_not_served( + json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}), + "gpt-4o", + json!({"litellm_provider": "openai"}) + )] + #[case::key_the_fetched_map_leaves_out_is_not_served(gpt_4o(2.0), "retired", Value::Null)] + #[case::new_key_is_served( + json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}), + "new", + json!({"litellm_provider": "openai", "input_cost_per_token": 5.0}) + )] + #[case::alias_is_served_without_the_aliases_field( + json!({"claude": {"litellm_provider": "anthropic", "input_cost_per_token": 4.0, "aliases": ["claude-latest"]}}), + "claude-latest", + json!({"litellm_provider": "anthropic", "input_cost_per_token": 4.0}) + )] + #[case::sample_spec_is_never_served( + json!({"sample_spec": {"input_cost_per_token": 0.0}, "gpt-4o": {"litellm_provider": "openai"}}), + "sample_spec", + Value::Null + )] + #[case::rejected_row_is_served_from_bundled( + gpt_4o(-1.0), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384}) + )] + #[case::rejected_row_with_no_bundled_row_is_left_out( + json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": "free"}}), + "internal/model", + Value::Null + )] + #[case::custom_map_serves_only_its_own_rows( + json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}), + "gpt-4o", + Value::Null + )] + fn layering(#[case] fetched: Value, #[case] key: &str, #[case] expected: Value) { + let mut world = World::new(); + world.install(&fetched); + assert_eq!(world.row(key), expected); + } + + #[rstest] + fn an_alias_and_its_owner_share_one_row_object() { + let mut world = World::new(); + world.install( + &json!({"claude": {"litellm_provider": "anthropic", "aliases": ["claude-latest"]}}), + ); + assert!(world.dict.shares_row("claude", "claude-latest")); + } + + #[rstest] + fn a_documents_own_rows_are_what_install_serves() { + let bundled = Arc::new(parse(&bundled_document())); + let document = fetched(&gpt_4o(2.0), &bundled); + let mut installs = Installs::new(bundled); + let composed = installs.begin(Arc::clone(&document), 1).compose(); + let own: IndexMap<&str, &Fields> = document.served().collect(); + let served: IndexMap<&str, &Fields> = composed + .iter() + .map(|(key, fields)| (key.as_str(), fields)) + .collect(); + assert_eq!(own, served); + } + + #[rstest] + #[case::below_the_minimum_model_count(10, 0.0, Some("below_minimum"))] + #[case::shrunk_against_the_bundled_count(1, 0.9, Some("shrunk"))] + #[case::within_both_limits(1, 0.3, None)] + fn whole_map_checks( + #[case] min_model_count: usize, + #[case] max_shrink_ratio: f64, + #[case] refused: Option<&str>, + ) { + let bundled = parse(&bundled_document()); + let kind = parse(&gpt_4o(2.0)) + .check_size( + bundled.model_count(), + IntegrityLimits::new(min_model_count, max_shrink_ratio).unwrap(), + ) + .err() + .map(|located| match located.error { + Error::BelowMinimum { .. } => "below_minimum", + Error::Shrunk { .. } => "shrunk", + other => panic!("unexpected {other}"), + }); + assert_eq!(kind, refused); + } + + #[rstest] + #[case::user_value_is_kept_and_untouched_fields_follow( + Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)), + gpt_4o_row(2.0, 32000), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 9.0, "max_tokens": 32000}) + )] + #[case::user_deleted_field_stays_deleted( + Edit::Unset("gpt-4o", "max_tokens"), + gpt_4o_row(1.0, 32000), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 1.0}) + )] + #[case::user_deleted_key_is_not_added_back( + Edit::Pop("gpt-4o"), + gpt_4o_row(2.0, 16384), + "gpt-4o", + Value::Null + )] + #[case::user_replaced_row_keeps_only_the_users_fields( + Edit::Insert("gpt-4o", json!({"input_cost_per_token": 9.0})), + gpt_4o_row(2.0, 32000), + "gpt-4o", + json!({"input_cost_per_token": 9.0}) + )] + #[case::user_key_no_layer_has_is_untouched( + Edit::Insert("mine", json!({"input_cost_per_token": 9.0})), + gpt_4o_row(2.0, 16384), + "mine", + json!({"input_cost_per_token": 9.0}) + )] + #[case::user_key_a_later_fetch_brings_keeps_the_users_fields( + Edit::Insert("new", json!({"input_cost_per_token": 9.0})), + json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 5}}), + "new", + json!({"input_cost_per_token": 9.0, "litellm_provider": "openai", "max_tokens": 5}) + )] + #[case::key_the_fetched_map_drops_keeps_only_the_users_fields( + Edit::Set("retired", "note", json!("mine")), + gpt_4o_row(2.0, 16384), + "retired", + json!({"note": "mine"}) + )] + #[case::foreign_value_is_kept( + Edit::SetForeign("gpt-4o", "input_cost_per_token"), + gpt_4o_row(2.0, 16384), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": FOREIGN, "max_tokens": 16384}) + )] + #[case::non_row_value_is_kept(Edit::ReplaceWithNonRow("gpt-4o"), gpt_4o_row(2.0, 16384), "gpt-4o", json!(NOT_A_ROW))] + #[case::user_write_equal_to_litellms_value_follows_later_fetches( + Edit::Set("gpt-4o", "input_cost_per_token", json!(1.0)), + gpt_4o_row(2.0, 16384), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384}) + )] + #[case::rebind_is_written_in_full( + Edit::RebindToCopy { dict_id: 2 }, + gpt_4o_row(2.0, 16384), + "gpt-4o", + json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384}) + )] + fn direct_writes( + #[case] edit: Edit, + #[case] fetched: Value, + #[case] key: &str, + #[case] expected: Value, + ) { + let mut world = World::new(); + world.edit(&edit); + world.install(&fetched); + assert_eq!(world.row(key), expected); + } + + #[rstest] + fn rebind_drops_direct_writes() { + let mut world = World::new(); + world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0))); + world.edit(&Edit::Insert("mine", json!({"input_cost_per_token": 9.0}))); + world.edit(&Edit::RebindToCopy { dict_id: 2 }); + world.install(&gpt_4o(2.0)); + assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(2.0)); + assert_eq!(world.row("mine"), Value::Null); + } + + #[rstest] + fn overlapping_installs_serve_the_last_one_begun_and_keep_following_fetches() { + let mut world = World::new(); + let first = world.begin(&gpt_4o(2.0)); + let second = world.begin(&gpt_4o(3.0)); + let first_outcome = world.write(first); + world.installs.finish(first_outcome.record); + let second_outcome = world.write(second); + world.installs.finish(second_outcome.record); + assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(3.0)); + + world.install(&gpt_4o(4.0)); + assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(4.0)); + } +} + +/// Not decided yet. Each case pins what the code does today, so deciding one changes its expected value +mod open { + use super::*; + + #[rstest] + fn a_rebind_to_a_dict_at_the_old_address_keeps_direct_writes() { + let mut world = World::new(); + world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0))); + world.edit(&Edit::RebindToCopy { dict_id: 1 }); + world.install(&gpt_4o(2.0)); + assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(9.0)); + } +} diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index f4c44a8f5ed..e010ea45846 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, LazyLock}; use litellm_host_python::{release_gil, to_py}; use litellm_model_catalog::wire::Fields; -use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers}; +use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Installs, IntegrityLimits}; use parking_lot::Mutex; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; @@ -16,7 +16,7 @@ use pyo3::types::{PyBytes, PyDict, PyModule}; use serde::Serialize; use sha1::{Digest, Sha1}; -use install::{Record, write_into}; +use install::BoundDict; pub(crate) use row_error::{RowError, RowErrorKind, row_error}; static BUNDLED: LazyLock> = LazyLock::new(|| { @@ -72,16 +72,11 @@ pub(crate) struct Unavailable { reason: String, } -struct State { - layers: Layers, - record: Option, -} - #[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")] pub(crate) struct CatalogHandle { module: Py, attr: String, - state: Mutex, + installs: Mutex, } #[pymethods] @@ -126,10 +121,7 @@ impl CatalogHandle { Self { module: module.unbind(), attr, - state: Mutex::new(State { - layers: Layers::new(Arc::clone(&BUNDLED.catalog)), - record: None, - }), + installs: Mutex::new(Installs::new(Arc::clone(&BUNDLED.catalog))), } } @@ -177,20 +169,13 @@ impl CatalogHandle { .cast::() .map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?; let dict_id = dict.as_ptr() as usize; - let (layers, record) = { - let mut state = self.state.lock(); - state - .layers - .replace_fetched(Arc::clone(&parsed.contents.catalog)); - let record = state - .record - .clone() - .filter(|record| record.dict_id == dict_id); - (state.layers.clone(), record) - }; - let composed = release_gil(py, || layers.compose()); - let outcome = write_into(py, dict, &composed, &layers, record)?; - self.state.lock().record = Some(outcome.record); + let pending = self + .installs + .lock() + .begin(Arc::clone(&parsed.contents.catalog), dict_id); + let composed = release_gil(py, || pending.compose()); + let outcome = pending.write(&mut BoundDict::new(dict), &composed)?; + self.installs.lock().finish(outcome.record); Ok(Installed { count: composed.len(), written: outcome.written, diff --git a/litellm-rust/crates/python-bridge/src/model_registry/install.rs b/litellm-rust/crates/python-bridge/src/model_registry/install.rs index 0388cafeadb..699f78d0cb9 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/install.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/install.rs @@ -1,234 +1,116 @@ use std::collections::HashMap; -use indexmap::IndexMap; use litellm_host_python::{from_py, to_py}; -use litellm_model_catalog::Layers; use litellm_model_catalog::wire::Fields; +use litellm_model_catalog::{BoundMap, Held, Slot}; use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::Value; -/// The field values litellm last wrote into one dict object -#[derive(Clone, Debug)] -pub(crate) struct Record { - pub(crate) dict_id: usize, - rows: HashMap, +/// The dict bound to `litellm.model_cost` for one install, under one GIL hold +pub(crate) struct BoundDict<'a, 'py> { + dict: &'a Bound<'py, PyDict>, + /// The row object each alias group shares in this install, as main's alias expansion gives them + groups: HashMap>, } -pub(crate) struct Outcome { - pub(crate) record: Record, - pub(crate) written: usize, - pub(crate) removed: usize, -} - -/// What the dict holds for one field, as litellm compares it -#[derive(PartialEq)] -enum Held { - Absent, - Json(Value), - /// A value that does not convert to JSON, which only the user can have written - Foreign, -} - -/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the -/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the -/// last install is written in full, which is main's result after rebind then adopt -pub(crate) fn write_into( - py: Python<'_>, - dict: &Bound<'_, PyDict>, - composed: &IndexMap, - layers: &Layers, - previous: Option, -) -> PyResult { - let dict_id = dict.as_ptr() as usize; - let Some(previous) = previous else { - return write_in_full(py, dict, composed, layers, dict_id); - }; - let mut rows = previous.rows; - let mut written = 0; - let mut removed = 0; - let mut fresh: HashMap> = HashMap::new(); - let keys: Vec = composed - .keys() - .chain(rows.keys().filter(|key| !composed.contains_key(*key))) - .cloned() - .collect(); - for key in keys { - let target = composed.get(&key); - let last = rows.remove(&key); - let Some(current) = dict.get_item(&key)? else { - match (target, last) { - (Some(target), None) => { - dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?; - rows.insert(key, target.clone()); - written += 1; - } - (_, Some(last)) if target.is_some() => { - rows.insert(key, last); - } - _ => {} - } - continue; - }; - let Ok(row) = current.cast::() else { - if let Some(last) = last { - rows.insert(key, last); - } - continue; - }; - let last = last.unwrap_or_default(); - let (owned, changed) = write_owned_fields(py, row, target, &last)?; - if target.is_none() && changed && row.is_empty() { - dict.del_item(&key)?; - removed += 1; - continue; - } - written += usize::from(changed); - if !owned.is_empty() { - rows.insert(key, owned); +impl<'a, 'py> BoundDict<'a, 'py> { + pub(crate) fn new(dict: &'a Bound<'py, PyDict>) -> Self { + Self { + dict, + groups: HashMap::new(), } } - Ok(Outcome { - record: Record { dict_id, rows }, - written, - removed, - }) + + fn py(&self) -> Python<'py> { + self.dict.py() + } + + fn row(&self, key: &str) -> PyResult> { + let row = self + .dict + .get_item(key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(key.to_owned()))?; + Ok(row.cast_into::()?) + } } -fn write_in_full( - py: Python<'_>, - dict: &Bound<'_, PyDict>, - composed: &IndexMap, - layers: &Layers, - dict_id: usize, -) -> PyResult { - let stale: Vec> = dict - .keys() - .iter() - .filter(|key| { - key.extract::() - .map_or(true, |key| !composed.contains_key(&key)) +fn held_value(value: &Bound<'_, PyAny>) -> Held { + from_py::(value).map_or(Held::Foreign, Held::Json) +} + +impl BoundMap for BoundDict<'_, '_> { + type Error = PyErr; + + fn slot(&self, key: &str) -> PyResult { + Ok(match self.dict.get_item(key)? { + None => Slot::Missing, + Some(value) if value.is_instance_of::() => Slot::Row, + Some(_) => Slot::NotARow, }) - .collect(); - let removed = stale.len(); - for key in stale { - dict.del_item(key)?; } - let mut fresh: HashMap> = HashMap::new(); - let mut written = 0; - for (key, target) in composed { - let current = dict.get_item(key)?; - let row = current.as_ref().and_then(|row| row.cast::().ok()); - match row { - Some(row) => { - let owned_by_litellm: Fields = row - .iter() - .filter_map(|(field, value)| { - let field = field.extract::().ok()?; - Some((field, from_py::(&value).ok()?)) - }) - .collect(); - let extra: Vec = row - .keys() - .iter() - .filter_map(|field| field.extract::().ok()) - .filter(|field| !owned_by_litellm.contains_key(field)) - .collect(); - for field in extra { - row.del_item(field)?; - } - let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?; - written += usize::from(changed); - } + + fn held(&self, key: &str, field: &str) -> PyResult { + Ok(self + .row(key)? + .get_item(field)? + .map_or(Held::Absent, |value| held_value(&value))) + } + + fn fields(&self, key: &str) -> PyResult> { + Ok(self + .row(key)? + .iter() + .filter_map(|(field, value)| { + Some((field.extract::().ok()?, held_value(&value))) + }) + .collect()) + } + + fn row_is_empty(&self, key: &str) -> PyResult { + Ok(self.row(key)?.is_empty()) + } + + fn set_field(&mut self, key: &str, field: &str, value: &Value) -> PyResult<()> { + self.row(key)?.set_item(field, to_py(self.py(), value)?) + } + + fn del_field(&mut self, key: &str, field: &str) -> PyResult<()> { + self.row(key)?.del_item(field) + } + + fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> PyResult<()> { + let py = self.py(); + let row = match self.groups.get(group) { + Some(row) => row.clone_ref(py), None => { - dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?; - written += 1; - } - } - } - Ok(Outcome { - record: Record { - dict_id, - rows: composed - .iter() - .map(|(key, fields)| (key.clone(), fields.clone())) - .collect(), - }, - written, - removed, - }) -} - -/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed -fn write_owned_fields( - py: Python<'_>, - row: &Bound<'_, PyDict>, - target: Option<&Fields>, - last: &Fields, -) -> PyResult<(Fields, bool)> { - let empty = Fields::new(); - let target = target.unwrap_or(&empty); - let fields: Vec<&String> = target - .keys() - .chain(last.keys().filter(|field| !target.contains_key(*field))) - .collect(); - let mut owned = Fields::new(); - let mut changed = false; - for field in fields { - let held = held(row, field)?; - let litellm_owns = match (&held, last.get(field)) { - (Held::Absent, None) => true, - (Held::Json(value), Some(last)) => value == last, - _ => false, - }; - if !litellm_owns { - if let Some(last) = last.get(field) { - owned.insert(field.clone(), last.clone()); - } - continue; - } - match target.get(field) { - Some(value) => { - if held != Held::Json(value.clone()) { + let row = PyDict::new(py); + for (field, value) in fields { row.set_item(field, to_py(py, value)?)?; - changed = true; } - owned.insert(field.clone(), value.clone()); + let row = row.unbind(); + self.groups.insert(group.to_owned(), row.clone_ref(py)); + row } - None if held != Held::Absent => { - row.del_item(field)?; - changed = true; - } - None => {} + }; + self.dict.set_item(key, row) + } + + fn del_key(&mut self, key: &str) -> PyResult<()> { + self.dict.del_item(key) + } + + fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> PyResult { + let stale: Vec> = self + .dict + .keys() + .iter() + .filter(|key| key.extract::().map_or(true, |key| !keep(&key))) + .collect(); + let removed = stale.len(); + for key in stale { + self.dict.del_item(key)?; } + Ok(removed) } - Ok((owned, changed)) -} - -fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult { - Ok(match row.get_item(field)? { - None => Held::Absent, - Some(value) => from_py::(&value).map_or(Held::Foreign, Held::Json), - }) -} - -/// A new row, the same object for every key of one alias group, as main's alias expansion gives them -fn shared_row( - py: Python<'_>, - fresh: &mut HashMap>, - layers: &Layers, - key: &str, - fields: &Fields, -) -> PyResult> { - let owner = layers.alias_owner(key).unwrap_or(key).to_owned(); - if let Some(row) = fresh.get(&owner) { - return Ok(row.clone_ref(py)); - } - let row = PyDict::new(py); - for (field, value) in fields { - row.set_item(field, to_py(py, value)?)?; - } - let row = row.unbind(); - fresh.insert(owner, row.clone_ref(py)); - Ok(row) } diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index cfb24580fac..c14ad8c9580 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -109,9 +109,7 @@ def test_evaluation_and_decisions_rows_with_zero_or_missing_output_price_pass(mo def test_chat_rows_with_output_price_pass() -> None: - head: Final = _snapshot( - {**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)} - ) + head: Final = _snapshot({**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)}) assert _failures(head) == () @@ -186,32 +184,50 @@ def test_bot_may_only_touch_the_cost_map_files() -> None: ) -_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it" -_REMOVED_VISION: Final = ( - "openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it" -) +_REMOVED_B: Final = "openrouter/b is removed before a past deprecation_date; set deprecation_date first" +_PAST: Final = "2000-01-01" +_FUTURE: Final = "2999-01-01" -def test_bot_may_not_remove_models() -> None: +def _base_with(**rows: object) -> object: + return _snapshot({**BASE_MAP, **rows}) + + +def test_a_human_pr_may_not_remove_a_model_without_a_past_deprecation_date() -> None: head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"}) assert _failures(head, bot=False) == (_REMOVED_B,) assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b") +@pytest.mark.parametrize( + ("deprecation_date", "expected"), + [(_PAST, ()), (_FUTURE, (_REMOVED_B,))], + ids=["past", "future"], +) +def test_a_model_is_removable_once_its_base_deprecation_date_has_passed( + deprecation_date: str, expected: tuple[str, ...] +) -> None: + base = _base_with(**{"openrouter/b": _entry(2e-06, deprecation_date=deprecation_date)}) + head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"}) + failures = guard.guard_failures(base, head, MAP_FILES, False, evaluation_output_price_sources={}) + assert failures == expected + + +def test_a_null_row_is_refused_since_released_wheels_crash_on_it() -> None: + head = _snapshot({**BASE_MAP, "openrouter/b": None}) + assert any("openrouter/b" in failure for failure in _failures(head, bot=False)) + + +@pytest.mark.parametrize("field", ["supports_vision", "max_tokens", "input_cost_per_token"]) +def test_a_human_pr_may_remove_any_field(field: str) -> None: + head = _snapshot({**BASE_MAP, "openrouter/a": {k: v for k, v in BASE_MAP["openrouter/a"].items() if k != field}}) + assert _failures(head, bot=False) == () + + def test_bot_may_not_remove_fields() -> None: head = _snapshot({**BASE_MAP, "openrouter/a": _entry()}) - assert _failures(head, bot=False) == (_REMOVED_VISION,) - assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision") - - -def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None: - head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None}) - assert _failures(head, bot=False) == () - - -def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None: - head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}}) assert _failures(head, bot=False) == () + assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",) def test_bot_may_not_change_special_root_keys() -> None: