This commit is contained in:
Yujong Lee 2026-10-28 12:17:12 -07:00 • committed by yujonglee
parent d5155204cd
commit 61c28232db
8 changed files with 923 additions and 363 deletions

View file

@ -564,51 +564,4 @@ mod tests {
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
);
}
use serde_json::json;
fn parse(document: Value) -> Loaded<Catalog> {
Catalog::parse(document.to_string().as_bytes()).unwrap()
}
#[rstest]
fn a_rejected_row_is_replaced_by_bundled_or_left_out() {
let bundled =
parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}}))
.value;
let fetched = parse(json!({
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1},
"acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"},
"kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0}
}));
let (catalog, rejections) = fetched
.value
.with_rejected_replaced(&fetched.errors, &bundled);
let replaced: Vec<(&str, bool)> = rejections
.iter()
.map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled))
.collect();
assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]);
assert_eq!(
catalog
.lookup("gpt-4o")
.unwrap()
.entry
.field("input_cost_per_token"),
Some(&json!(1.0))
);
assert_eq!(catalog.lookup("GPT-4O").unwrap().matched_key, "gpt-4o");
assert!(catalog.lookup("acme/new").is_none());
assert!(catalog.lookup("ACME/NEW").is_none());
assert!(catalog.lookup("kept").is_some());
}
#[rstest]
fn a_null_row_is_a_removal_not_an_error() {
let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
assert!(loaded.errors.is_empty(), "{:?}", loaded.errors);
assert_eq!(loaded.value.removals().collect::<Vec<_>>(), ["retired"]);
assert_eq!(loaded.value.served().count(), 1);
}
}

View file

@ -0,0 +1,271 @@
use std::collections::HashMap;
use std::sync::Arc;
use indexmap::IndexMap;
use serde_json::Value;
use crate::catalog::Catalog;
use crate::layers::Layers;
use crate::wire::Fields;
/// What the bound dict holds for one field, as litellm compares it
#[derive(Clone, Debug, PartialEq)]
pub enum Held {
Absent,
Json(Value),
/// A value that does not convert to JSON, which only the user can have written
Foreign,
}
/// What the bound dict holds under one key
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Slot {
Missing,
/// Anything but a dict, which only the user can have written
NotARow,
Row,
}
/// The dict install writes into, `litellm.model_cost` in the Python host
pub trait BoundMap {
type Error;
fn slot(&self, key: &str) -> Result<Slot, Self::Error>;
/// Called only for a key whose slot is `Row`
fn held(&self, key: &str, field: &str) -> Result<Held, Self::Error>;
/// The row's string-named fields, called only for a key whose slot is `Row`
fn fields(&self, key: &str) -> Result<Vec<(String, Held)>, Self::Error>;
fn row_is_empty(&self, key: &str) -> Result<bool, Self::Error>;
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Self::Error>;
fn del_field(&mut self, key: &str, field: &str) -> Result<(), Self::Error>;
/// A new row under `key`, the same object for every key given the same `group` in one install
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Self::Error>;
fn del_key(&mut self, key: &str) -> Result<(), Self::Error>;
/// Deletes every key, string or not, that `keep` refuses, and returns how many
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result<usize, Self::Error>;
}
/// The field values litellm last wrote into one dict object
#[derive(Clone, Debug)]
pub struct Record {
dict_id: usize,
rows: HashMap<String, Fields>,
}
#[derive(Debug)]
pub struct Outcome {
pub record: Record,
/// Keys set because the row differed or was new
pub written: usize,
/// Keys deleted because the map no longer serves them
pub removed: usize,
}
/// The layers and what the last install wrote, kept between installs
#[derive(Debug)]
pub struct Installs {
layers: Layers,
record: Option<Record>,
}
/// One install between `Installs::begin` and `Installs::finish`
#[derive(Debug)]
pub struct Pending {
layers: Layers,
previous: Option<Record>,
dict_id: usize,
}
impl Installs {
pub fn new(bundled: Arc<Catalog>) -> Self {
Self {
layers: Layers::new(bundled),
record: None,
}
}
/// Replaces the fetched layer. The last record applies only while `dict_id` names the dict it was written into
pub fn begin(&mut self, fetched: Arc<Catalog>, dict_id: usize) -> Pending {
self.layers.replace_fetched(fetched);
Pending {
layers: self.layers.clone(),
previous: self
.record
.clone()
.filter(|record| record.dict_id == dict_id),
dict_id,
}
}
pub fn finish(&mut self, record: Record) {
self.record = Some(record);
}
}
impl Pending {
pub fn compose(&self) -> IndexMap<String, Fields> {
self.layers.compose()
}
/// A field is written only while litellm still owns it, meaning the dict holds the value last written there, or
/// nothing where nothing was written. Without a record for this dict the dict is written in full, which is main's
/// result after rebind then adopt
pub fn write<M: BoundMap>(
self,
map: &mut M,
composed: &IndexMap<String, Fields>,
) -> Result<Outcome, M::Error> {
match self.previous {
Some(previous) => write_owned(map, composed, &self.layers, previous),
None => write_in_full(map, composed, &self.layers, self.dict_id),
}
}
}
fn write_owned<M: BoundMap>(
map: &mut M,
composed: &IndexMap<String, Fields>,
layers: &Layers,
previous: Record,
) -> Result<Outcome, M::Error> {
let mut rows = previous.rows;
let mut written = 0;
let mut removed = 0;
let keys: Vec<String> = composed
.keys()
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
.cloned()
.collect();
for key in keys {
let target = composed.get(&key);
let last = rows.remove(&key);
match map.slot(&key)? {
Slot::Missing => match (target, last) {
(Some(target), None) => {
map.insert_row(&key, group(layers, &key), target)?;
rows.insert(key, target.clone());
written += 1;
}
(Some(_), Some(last)) => {
rows.insert(key, last);
}
(None, _) => {}
},
Slot::NotARow => {
if let Some(last) = last {
rows.insert(key, last);
}
}
Slot::Row => {
let last = last.unwrap_or_default();
let (owned, changed) = write_owned_fields(map, &key, target, &last)?;
if target.is_none() && changed && map.row_is_empty(&key)? {
map.del_key(&key)?;
removed += 1;
continue;
}
written += usize::from(changed);
if !owned.is_empty() {
rows.insert(key, owned);
}
}
}
}
Ok(Outcome {
record: Record {
dict_id: previous.dict_id,
rows,
},
written,
removed,
})
}
fn write_in_full<M: BoundMap>(
map: &mut M,
composed: &IndexMap<String, Fields>,
layers: &Layers,
dict_id: usize,
) -> Result<Outcome, M::Error> {
let removed = map.retain_keys(&|key| composed.contains_key(key))?;
let mut written = 0;
for (key, target) in composed {
if map.slot(key)? != Slot::Row {
map.insert_row(key, group(layers, key), target)?;
written += 1;
continue;
}
let mut current = Fields::new();
for (field, held) in map.fields(key)? {
match held {
Held::Json(value) => {
current.insert(field, value);
}
Held::Absent | Held::Foreign => map.del_field(key, &field)?,
}
}
let (_, changed) = write_owned_fields(map, key, Some(target), &current)?;
written += usize::from(changed);
}
Ok(Outcome {
record: Record {
dict_id,
rows: composed
.iter()
.map(|(key, fields)| (key.clone(), fields.clone()))
.collect(),
},
written,
removed,
})
}
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
fn write_owned_fields<M: BoundMap>(
map: &mut M,
key: &str,
target: Option<&Fields>,
last: &Fields,
) -> Result<(Fields, bool), M::Error> {
let empty = Fields::new();
let target = target.unwrap_or(&empty);
let fields: Vec<&String> = target
.keys()
.chain(last.keys().filter(|field| !target.contains_key(*field)))
.collect();
let mut owned = Fields::new();
let mut changed = false;
for field in fields {
let held = map.held(key, field)?;
let litellm_owns = match (&held, last.get(field)) {
(Held::Absent, None) => true,
(Held::Json(value), Some(last)) => value == last,
_ => false,
};
if !litellm_owns {
if let Some(last) = last.get(field) {
owned.insert(field.clone(), last.clone());
}
continue;
}
match target.get(field) {
Some(value) => {
if held != Held::Json(value.clone()) {
map.set_field(key, field, value)?;
changed = true;
}
owned.insert(field.clone(), value.clone());
}
None if held != Held::Absent => {
map.del_field(key, field)?;
changed = true;
}
None => {}
}
}
Ok((owned, changed))
}
fn group<'a>(layers: &'a Layers, key: &'a str) -> &'a str {
layers.alias_owner(key).unwrap_or(key)
}

View file

@ -101,63 +101,3 @@ fn without_aliases(fields: &Fields) -> Fields {
.map(|(field, value)| (field.clone(), value.clone()))
.collect()
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn catalog(document: Value) -> Arc<Catalog> {
Arc::new(
Catalog::parse(document.to_string().as_bytes())
.unwrap()
.value,
)
}
fn bundled() -> Arc<Catalog> {
catalog(json!({
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0}
}))
}
fn composed(fetched: Value) -> IndexMap<String, Fields> {
let mut layers = Layers::new(bundled());
layers.replace_fetched(catalog(fetched));
layers.compose()
}
#[rstest]
fn a_field_the_fetched_row_lacks_is_served_from_bundled() {
let rows = composed(
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}),
);
assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0));
assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384));
}
#[rstest]
fn a_null_field_removes_it() {
let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}));
assert!(!rows["gpt-4o"].contains_key("max_tokens"));
}
#[rstest]
fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() {
let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}}));
let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
assert!(kept.contains_key("retired"));
assert!(!retired.contains_key("retired"));
}
#[rstest]
fn aliases_are_served_under_every_name_without_the_aliases_field() {
let rows =
composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}}));
assert_eq!(rows["new-alias"], rows["new"]);
assert!(!rows["new"].contains_key("aliases"));
}
}

View file

@ -39,6 +39,7 @@ mod bundled;
mod catalog;
mod error;
mod fallback;
mod install;
mod layers;
mod lower;
mod model;
@ -53,6 +54,7 @@ pub use bundled::*;
pub use catalog::*;
pub use error::*;
pub use fallback::*;
pub use install::*;
pub use layers::*;
pub use model::*;
pub use model_info::*;

View file

@ -0,0 +1,511 @@
use std::collections::HashMap;
use std::convert::Infallible;
use std::sync::Arc;
use indexmap::IndexMap;
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{
BoundMap, Catalog, Error, Held, Installs, IntegrityLimits, Outcome, Pending, Slot,
};
use rstest::rstest;
use serde_json::{Value, json};
const FOREIGN: &str = "<foreign>";
const NOT_A_ROW: &str = "<not a row>";
fn bundled_document() -> Value {
json!({
"sample_spec": {"litellm_provider": "one of the providers", "input_cost_per_token": 0.0},
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0},
"claude": {"litellm_provider": "anthropic", "input_cost_per_token": 3.0, "aliases": ["claude-latest"]}
})
}
fn parse(document: &Value) -> Catalog {
Catalog::parse(document.to_string().as_bytes())
.unwrap()
.value
}
/// What python-bridge's `parse` hands to install for a fetched body
fn fetched(document: &Value, bundled: &Catalog) -> Arc<Catalog> {
let loaded = Catalog::parse(document.to_string().as_bytes()).unwrap();
Arc::new(
loaded
.value
.with_rejected_replaced(&loaded.errors, bundled)
.0,
)
}
#[derive(Clone, Debug, PartialEq)]
enum Stored {
Json(Value),
Foreign,
}
#[derive(Clone, Debug)]
enum Entry {
Row(usize),
NotARow,
}
/// A Python dict of rows, where two keys can hold the same row object
#[derive(Clone, Debug, Default)]
struct Dict {
keys: IndexMap<String, Entry>,
rows: Vec<IndexMap<String, Stored>>,
}
struct Session<'a> {
dict: &'a mut Dict,
groups: HashMap<String, usize>,
}
impl Dict {
fn row_mut(&mut self, key: &str) -> &mut IndexMap<String, Stored> {
match self.keys[key] {
Entry::Row(index) => &mut self.rows[index],
Entry::NotARow => panic!("{key} is not a row"),
}
}
fn view(&self, key: &str) -> Value {
match self.keys.get(key) {
None => Value::Null,
Some(Entry::NotARow) => json!(NOT_A_ROW),
Some(Entry::Row(index)) => self.rows[*index]
.iter()
.map(|(field, stored)| {
let value = match stored {
Stored::Json(value) => value.clone(),
Stored::Foreign => json!(FOREIGN),
};
(field.clone(), value)
})
.collect(),
}
}
fn shares_row(&self, a: &str, b: &str) -> bool {
matches!(
(&self.keys[a], &self.keys[b]),
(Entry::Row(a), Entry::Row(b)) if a == b
)
}
}
impl BoundMap for Session<'_> {
type Error = Infallible;
fn slot(&self, key: &str) -> Result<Slot, Infallible> {
Ok(match self.dict.keys.get(key) {
None => Slot::Missing,
Some(Entry::NotARow) => Slot::NotARow,
Some(Entry::Row(_)) => Slot::Row,
})
}
fn held(&self, key: &str, field: &str) -> Result<Held, Infallible> {
let Entry::Row(index) = self.dict.keys[key] else {
panic!("{key} is not a row");
};
Ok(match self.dict.rows[index].get(field) {
None => Held::Absent,
Some(Stored::Json(value)) => Held::Json(value.clone()),
Some(Stored::Foreign) => Held::Foreign,
})
}
fn fields(&self, key: &str) -> Result<Vec<(String, Held)>, Infallible> {
let Entry::Row(index) = self.dict.keys[key] else {
panic!("{key} is not a row");
};
Ok(self.dict.rows[index]
.iter()
.map(|(field, stored)| {
let held = match stored {
Stored::Json(value) => Held::Json(value.clone()),
Stored::Foreign => Held::Foreign,
};
(field.clone(), held)
})
.collect())
}
fn row_is_empty(&self, key: &str) -> Result<bool, Infallible> {
Ok(matches!(self.dict.keys[key], Entry::Row(index) if self.dict.rows[index].is_empty()))
}
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Infallible> {
self.dict
.row_mut(key)
.insert(field.to_owned(), Stored::Json(value.clone()));
Ok(())
}
fn del_field(&mut self, key: &str, field: &str) -> Result<(), Infallible> {
self.dict.row_mut(key).shift_remove(field);
Ok(())
}
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Infallible> {
let index = match self.groups.get(group) {
Some(index) => *index,
None => {
self.dict.rows.push(
fields
.iter()
.map(|(field, value)| (field.clone(), Stored::Json(value.clone())))
.collect(),
);
let index = self.dict.rows.len() - 1;
self.groups.insert(group.to_owned(), index);
index
}
};
self.dict.keys.insert(key.to_owned(), Entry::Row(index));
Ok(())
}
fn del_key(&mut self, key: &str) -> Result<(), Infallible> {
self.dict.keys.shift_remove(key);
Ok(())
}
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result<usize, Infallible> {
let before = self.dict.keys.len();
self.dict.keys.retain(|key, _| keep(key));
Ok(before - self.dict.keys.len())
}
}
/// What the user does to `litellm.model_cost` between two installs
#[derive(Clone, Debug)]
enum Edit {
Set(&'static str, &'static str, Value),
Unset(&'static str, &'static str),
SetForeign(&'static str, &'static str),
Pop(&'static str),
Insert(&'static str, Value),
ReplaceWithNonRow(&'static str),
/// `litellm.model_cost = dict(litellm.model_cost)`, the copy landing at `dict_id`
RebindToCopy {
dict_id: usize,
},
}
/// One Python process: the bundled map installed at import, then the user and later installs
struct World {
bundled: Arc<Catalog>,
installs: Installs,
dict: Dict,
dict_id: usize,
}
impl World {
fn new() -> Self {
let bundled = Arc::new(parse(&bundled_document()));
let mut world = Self {
installs: Installs::new(Arc::clone(&bundled)),
bundled: Arc::clone(&bundled),
dict: Dict::default(),
dict_id: 1,
};
world.install_catalog(bundled);
world
}
fn begin(&mut self, document: &Value) -> Pending {
let catalog = fetched(document, &self.bundled);
self.installs.begin(catalog, self.dict_id)
}
fn write(&mut self, pending: Pending) -> Outcome {
let composed = pending.compose();
let mut session = Session {
dict: &mut self.dict,
groups: HashMap::new(),
};
let Ok(outcome) = pending.write(&mut session, &composed);
outcome
}
fn install_catalog(&mut self, catalog: Arc<Catalog>) {
let pending = self.installs.begin(catalog, self.dict_id);
let outcome = self.write(pending);
self.installs.finish(outcome.record);
}
fn install(&mut self, document: &Value) {
let catalog = fetched(document, &self.bundled);
self.install_catalog(catalog);
}
fn edit(&mut self, edit: &Edit) {
match edit {
Edit::Set(key, field, value) => {
self.dict
.row_mut(key)
.insert((*field).to_owned(), Stored::Json(value.clone()));
}
Edit::Unset(key, field) => {
self.dict.row_mut(key).shift_remove(*field);
}
Edit::SetForeign(key, field) => {
self.dict
.row_mut(key)
.insert((*field).to_owned(), Stored::Foreign);
}
Edit::Pop(key) => {
self.dict.keys.shift_remove(*key);
}
Edit::Insert(key, row) => {
self.dict.rows.push(
row.as_object()
.unwrap()
.iter()
.map(|(field, value)| (field.clone(), Stored::Json(value.clone())))
.collect(),
);
let index = self.dict.rows.len() - 1;
self.dict.keys.insert((*key).to_owned(), Entry::Row(index));
}
Edit::ReplaceWithNonRow(key) => {
self.dict.keys.insert((*key).to_owned(), Entry::NotARow);
}
Edit::RebindToCopy { dict_id } => {
self.dict = self.dict.clone();
self.dict_id = *dict_id;
}
}
}
fn row(&self, key: &str) -> Value {
self.dict.view(key)
}
}
fn gpt_4o(price: f64) -> Value {
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price}})
}
fn gpt_4o_row(price: f64, max_tokens: u32) -> Value {
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price, "max_tokens": max_tokens}})
}
/// The behavior users expect. A fetched map replaces bundled at row level, bundled is only the fallback for a bad
/// row or a bad document, and the user's own writes survive every install. Cases the code does not meet yet fail
mod expected {
use super::*;
#[rstest]
#[case::fetched_row_is_served_as_written(
gpt_4o(2.0),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0})
)]
#[case::null_field_is_not_served(
json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}),
"gpt-4o",
json!({"litellm_provider": "openai"})
)]
#[case::key_the_fetched_map_leaves_out_is_not_served(gpt_4o(2.0), "retired", Value::Null)]
#[case::new_key_is_served(
json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}),
"new",
json!({"litellm_provider": "openai", "input_cost_per_token": 5.0})
)]
#[case::alias_is_served_without_the_aliases_field(
json!({"claude": {"litellm_provider": "anthropic", "input_cost_per_token": 4.0, "aliases": ["claude-latest"]}}),
"claude-latest",
json!({"litellm_provider": "anthropic", "input_cost_per_token": 4.0})
)]
#[case::sample_spec_is_never_served(
json!({"sample_spec": {"input_cost_per_token": 0.0}, "gpt-4o": {"litellm_provider": "openai"}}),
"sample_spec",
Value::Null
)]
#[case::rejected_row_is_served_from_bundled(
gpt_4o(-1.0),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384})
)]
#[case::rejected_row_with_no_bundled_row_is_left_out(
json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": "free"}}),
"internal/model",
Value::Null
)]
#[case::custom_map_serves_only_its_own_rows(
json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}),
"gpt-4o",
Value::Null
)]
fn layering(#[case] fetched: Value, #[case] key: &str, #[case] expected: Value) {
let mut world = World::new();
world.install(&fetched);
assert_eq!(world.row(key), expected);
}
#[rstest]
fn an_alias_and_its_owner_share_one_row_object() {
let mut world = World::new();
world.install(
&json!({"claude": {"litellm_provider": "anthropic", "aliases": ["claude-latest"]}}),
);
assert!(world.dict.shares_row("claude", "claude-latest"));
}
#[rstest]
fn a_documents_own_rows_are_what_install_serves() {
let bundled = Arc::new(parse(&bundled_document()));
let document = fetched(&gpt_4o(2.0), &bundled);
let mut installs = Installs::new(bundled);
let composed = installs.begin(Arc::clone(&document), 1).compose();
let own: IndexMap<&str, &Fields> = document.served().collect();
let served: IndexMap<&str, &Fields> = composed
.iter()
.map(|(key, fields)| (key.as_str(), fields))
.collect();
assert_eq!(own, served);
}
#[rstest]
#[case::below_the_minimum_model_count(10, 0.0, Some("below_minimum"))]
#[case::shrunk_against_the_bundled_count(1, 0.9, Some("shrunk"))]
#[case::within_both_limits(1, 0.3, None)]
fn whole_map_checks(
#[case] min_model_count: usize,
#[case] max_shrink_ratio: f64,
#[case] refused: Option<&str>,
) {
let bundled = parse(&bundled_document());
let kind = parse(&gpt_4o(2.0))
.check_size(
bundled.model_count(),
IntegrityLimits::new(min_model_count, max_shrink_ratio).unwrap(),
)
.err()
.map(|located| match located.error {
Error::BelowMinimum { .. } => "below_minimum",
Error::Shrunk { .. } => "shrunk",
other => panic!("unexpected {other}"),
});
assert_eq!(kind, refused);
}
#[rstest]
#[case::user_value_is_kept_and_untouched_fields_follow(
Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)),
gpt_4o_row(2.0, 32000),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 9.0, "max_tokens": 32000})
)]
#[case::user_deleted_field_stays_deleted(
Edit::Unset("gpt-4o", "max_tokens"),
gpt_4o_row(1.0, 32000),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 1.0})
)]
#[case::user_deleted_key_is_not_added_back(
Edit::Pop("gpt-4o"),
gpt_4o_row(2.0, 16384),
"gpt-4o",
Value::Null
)]
#[case::user_replaced_row_keeps_only_the_users_fields(
Edit::Insert("gpt-4o", json!({"input_cost_per_token": 9.0})),
gpt_4o_row(2.0, 32000),
"gpt-4o",
json!({"input_cost_per_token": 9.0})
)]
#[case::user_key_no_layer_has_is_untouched(
Edit::Insert("mine", json!({"input_cost_per_token": 9.0})),
gpt_4o_row(2.0, 16384),
"mine",
json!({"input_cost_per_token": 9.0})
)]
#[case::user_key_a_later_fetch_brings_keeps_the_users_fields(
Edit::Insert("new", json!({"input_cost_per_token": 9.0})),
json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 5}}),
"new",
json!({"input_cost_per_token": 9.0, "litellm_provider": "openai", "max_tokens": 5})
)]
#[case::key_the_fetched_map_drops_keeps_only_the_users_fields(
Edit::Set("retired", "note", json!("mine")),
gpt_4o_row(2.0, 16384),
"retired",
json!({"note": "mine"})
)]
#[case::foreign_value_is_kept(
Edit::SetForeign("gpt-4o", "input_cost_per_token"),
gpt_4o_row(2.0, 16384),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": FOREIGN, "max_tokens": 16384})
)]
#[case::non_row_value_is_kept(Edit::ReplaceWithNonRow("gpt-4o"), gpt_4o_row(2.0, 16384), "gpt-4o", json!(NOT_A_ROW))]
#[case::user_write_equal_to_litellms_value_follows_later_fetches(
Edit::Set("gpt-4o", "input_cost_per_token", json!(1.0)),
gpt_4o_row(2.0, 16384),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384})
)]
#[case::rebind_is_written_in_full(
Edit::RebindToCopy { dict_id: 2 },
gpt_4o_row(2.0, 16384),
"gpt-4o",
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384})
)]
fn direct_writes(
#[case] edit: Edit,
#[case] fetched: Value,
#[case] key: &str,
#[case] expected: Value,
) {
let mut world = World::new();
world.edit(&edit);
world.install(&fetched);
assert_eq!(world.row(key), expected);
}
#[rstest]
fn rebind_drops_direct_writes() {
let mut world = World::new();
world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)));
world.edit(&Edit::Insert("mine", json!({"input_cost_per_token": 9.0})));
world.edit(&Edit::RebindToCopy { dict_id: 2 });
world.install(&gpt_4o(2.0));
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(2.0));
assert_eq!(world.row("mine"), Value::Null);
}
#[rstest]
fn overlapping_installs_serve_the_last_one_begun_and_keep_following_fetches() {
let mut world = World::new();
let first = world.begin(&gpt_4o(2.0));
let second = world.begin(&gpt_4o(3.0));
let first_outcome = world.write(first);
world.installs.finish(first_outcome.record);
let second_outcome = world.write(second);
world.installs.finish(second_outcome.record);
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(3.0));
world.install(&gpt_4o(4.0));
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(4.0));
}
}
/// Not decided yet. Each case pins what the code does today, so deciding one changes its expected value
mod open {
use super::*;
#[rstest]
fn a_rebind_to_a_dict_at_the_old_address_keeps_direct_writes() {
let mut world = World::new();
world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)));
world.edit(&Edit::RebindToCopy { dict_id: 1 });
world.install(&gpt_4o(2.0));
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(9.0));
}
}

View file

@ -6,7 +6,7 @@ use std::sync::{Arc, LazyLock};
use litellm_host_python::{release_gil, to_py};
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Installs, IntegrityLimits};
use parking_lot::Mutex;
use pyo3::exceptions::PyTypeError;
use pyo3::prelude::*;
@ -16,7 +16,7 @@ use pyo3::types::{PyBytes, PyDict, PyModule};
use serde::Serialize;
use sha1::{Digest, Sha1};
use install::{Record, write_into};
use install::BoundDict;
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
static BUNDLED: LazyLock<Arc<Contents>> = LazyLock::new(|| {
@ -72,16 +72,11 @@ pub(crate) struct Unavailable {
reason: String,
}
struct State {
layers: Layers,
record: Option<Record>,
}
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
pub(crate) struct CatalogHandle {
module: Py<PyModule>,
attr: String,
state: Mutex<State>,
installs: Mutex<Installs>,
}
#[pymethods]
@ -126,10 +121,7 @@ impl CatalogHandle {
Self {
module: module.unbind(),
attr,
state: Mutex::new(State {
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
record: None,
}),
installs: Mutex::new(Installs::new(Arc::clone(&BUNDLED.catalog))),
}
}
@ -177,20 +169,13 @@ impl CatalogHandle {
.cast::<PyDict>()
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?;
let dict_id = dict.as_ptr() as usize;
let (layers, record) = {
let mut state = self.state.lock();
state
.layers
.replace_fetched(Arc::clone(&parsed.contents.catalog));
let record = state
.record
.clone()
.filter(|record| record.dict_id == dict_id);
(state.layers.clone(), record)
};
let composed = release_gil(py, || layers.compose());
let outcome = write_into(py, dict, &composed, &layers, record)?;
self.state.lock().record = Some(outcome.record);
let pending = self
.installs
.lock()
.begin(Arc::clone(&parsed.contents.catalog), dict_id);
let composed = release_gil(py, || pending.compose());
let outcome = pending.write(&mut BoundDict::new(dict), &composed)?;
self.installs.lock().finish(outcome.record);
Ok(Installed {
count: composed.len(),
written: outcome.written,

View file

@ -1,234 +1,116 @@
use std::collections::HashMap;
use indexmap::IndexMap;
use litellm_host_python::{from_py, to_py};
use litellm_model_catalog::Layers;
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{BoundMap, Held, Slot};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::Value;
/// The field values litellm last wrote into one dict object
#[derive(Clone, Debug)]
pub(crate) struct Record {
pub(crate) dict_id: usize,
rows: HashMap<String, Fields>,
/// The dict bound to `litellm.model_cost` for one install, under one GIL hold
pub(crate) struct BoundDict<'a, 'py> {
dict: &'a Bound<'py, PyDict>,
/// The row object each alias group shares in this install, as main's alias expansion gives them
groups: HashMap<String, Py<PyDict>>,
}
pub(crate) struct Outcome {
pub(crate) record: Record,
pub(crate) written: usize,
pub(crate) removed: usize,
}
/// What the dict holds for one field, as litellm compares it
#[derive(PartialEq)]
enum Held {
Absent,
Json(Value),
/// A value that does not convert to JSON, which only the user can have written
Foreign,
}
/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the
/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the
/// last install is written in full, which is main's result after rebind then adopt
pub(crate) fn write_into(
py: Python<'_>,
dict: &Bound<'_, PyDict>,
composed: &IndexMap<String, Fields>,
layers: &Layers,
previous: Option<Record>,
) -> PyResult<Outcome> {
let dict_id = dict.as_ptr() as usize;
let Some(previous) = previous else {
return write_in_full(py, dict, composed, layers, dict_id);
};
let mut rows = previous.rows;
let mut written = 0;
let mut removed = 0;
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
let keys: Vec<String> = composed
.keys()
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
.cloned()
.collect();
for key in keys {
let target = composed.get(&key);
let last = rows.remove(&key);
let Some(current) = dict.get_item(&key)? else {
match (target, last) {
(Some(target), None) => {
dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?;
rows.insert(key, target.clone());
written += 1;
}
(_, Some(last)) if target.is_some() => {
rows.insert(key, last);
}
_ => {}
}
continue;
};
let Ok(row) = current.cast::<PyDict>() else {
if let Some(last) = last {
rows.insert(key, last);
}
continue;
};
let last = last.unwrap_or_default();
let (owned, changed) = write_owned_fields(py, row, target, &last)?;
if target.is_none() && changed && row.is_empty() {
dict.del_item(&key)?;
removed += 1;
continue;
}
written += usize::from(changed);
if !owned.is_empty() {
rows.insert(key, owned);
impl<'a, 'py> BoundDict<'a, 'py> {
pub(crate) fn new(dict: &'a Bound<'py, PyDict>) -> Self {
Self {
dict,
groups: HashMap::new(),
}
}
Ok(Outcome {
record: Record { dict_id, rows },
written,
removed,
})
fn py(&self) -> Python<'py> {
self.dict.py()
}
fn row(&self, key: &str) -> PyResult<Bound<'py, PyDict>> {
let row = self
.dict
.get_item(key)?
.ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(key.to_owned()))?;
Ok(row.cast_into::<PyDict>()?)
}
}
fn write_in_full(
py: Python<'_>,
dict: &Bound<'_, PyDict>,
composed: &IndexMap<String, Fields>,
layers: &Layers,
dict_id: usize,
) -> PyResult<Outcome> {
let stale: Vec<Bound<'_, PyAny>> = dict
.keys()
.iter()
.filter(|key| {
key.extract::<String>()
.map_or(true, |key| !composed.contains_key(&key))
fn held_value(value: &Bound<'_, PyAny>) -> Held {
from_py::<Value>(value).map_or(Held::Foreign, Held::Json)
}
impl BoundMap for BoundDict<'_, '_> {
type Error = PyErr;
fn slot(&self, key: &str) -> PyResult<Slot> {
Ok(match self.dict.get_item(key)? {
None => Slot::Missing,
Some(value) if value.is_instance_of::<PyDict>() => Slot::Row,
Some(_) => Slot::NotARow,
})
.collect();
let removed = stale.len();
for key in stale {
dict.del_item(key)?;
}
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
let mut written = 0;
for (key, target) in composed {
let current = dict.get_item(key)?;
let row = current.as_ref().and_then(|row| row.cast::<PyDict>().ok());
match row {
Some(row) => {
let owned_by_litellm: Fields = row
.iter()
.filter_map(|(field, value)| {
let field = field.extract::<String>().ok()?;
Some((field, from_py::<Value>(&value).ok()?))
})
.collect();
let extra: Vec<String> = row
.keys()
.iter()
.filter_map(|field| field.extract::<String>().ok())
.filter(|field| !owned_by_litellm.contains_key(field))
.collect();
for field in extra {
row.del_item(field)?;
}
let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?;
written += usize::from(changed);
}
fn held(&self, key: &str, field: &str) -> PyResult<Held> {
Ok(self
.row(key)?
.get_item(field)?
.map_or(Held::Absent, |value| held_value(&value)))
}
fn fields(&self, key: &str) -> PyResult<Vec<(String, Held)>> {
Ok(self
.row(key)?
.iter()
.filter_map(|(field, value)| {
Some((field.extract::<String>().ok()?, held_value(&value)))
})
.collect())
}
fn row_is_empty(&self, key: &str) -> PyResult<bool> {
Ok(self.row(key)?.is_empty())
}
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> PyResult<()> {
self.row(key)?.set_item(field, to_py(self.py(), value)?)
}
fn del_field(&mut self, key: &str, field: &str) -> PyResult<()> {
self.row(key)?.del_item(field)
}
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> PyResult<()> {
let py = self.py();
let row = match self.groups.get(group) {
Some(row) => row.clone_ref(py),
None => {
dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?;
written += 1;
}
}
}
Ok(Outcome {
record: Record {
dict_id,
rows: composed
.iter()
.map(|(key, fields)| (key.clone(), fields.clone()))
.collect(),
},
written,
removed,
})
}
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
fn write_owned_fields(
py: Python<'_>,
row: &Bound<'_, PyDict>,
target: Option<&Fields>,
last: &Fields,
) -> PyResult<(Fields, bool)> {
let empty = Fields::new();
let target = target.unwrap_or(&empty);
let fields: Vec<&String> = target
.keys()
.chain(last.keys().filter(|field| !target.contains_key(*field)))
.collect();
let mut owned = Fields::new();
let mut changed = false;
for field in fields {
let held = held(row, field)?;
let litellm_owns = match (&held, last.get(field)) {
(Held::Absent, None) => true,
(Held::Json(value), Some(last)) => value == last,
_ => false,
};
if !litellm_owns {
if let Some(last) = last.get(field) {
owned.insert(field.clone(), last.clone());
}
continue;
}
match target.get(field) {
Some(value) => {
if held != Held::Json(value.clone()) {
let row = PyDict::new(py);
for (field, value) in fields {
row.set_item(field, to_py(py, value)?)?;
changed = true;
}
owned.insert(field.clone(), value.clone());
let row = row.unbind();
self.groups.insert(group.to_owned(), row.clone_ref(py));
row
}
None if held != Held::Absent => {
row.del_item(field)?;
changed = true;
}
None => {}
};
self.dict.set_item(key, row)
}
fn del_key(&mut self, key: &str) -> PyResult<()> {
self.dict.del_item(key)
}
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> PyResult<usize> {
let stale: Vec<Bound<'_, PyAny>> = self
.dict
.keys()
.iter()
.filter(|key| key.extract::<String>().map_or(true, |key| !keep(&key)))
.collect();
let removed = stale.len();
for key in stale {
self.dict.del_item(key)?;
}
Ok(removed)
}
Ok((owned, changed))
}
fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult<Held> {
Ok(match row.get_item(field)? {
None => Held::Absent,
Some(value) => from_py::<Value>(&value).map_or(Held::Foreign, Held::Json),
})
}
/// A new row, the same object for every key of one alias group, as main's alias expansion gives them
fn shared_row(
py: Python<'_>,
fresh: &mut HashMap<String, Py<PyDict>>,
layers: &Layers,
key: &str,
fields: &Fields,
) -> PyResult<Py<PyDict>> {
let owner = layers.alias_owner(key).unwrap_or(key).to_owned();
if let Some(row) = fresh.get(&owner) {
return Ok(row.clone_ref(py));
}
let row = PyDict::new(py);
for (field, value) in fields {
row.set_item(field, to_py(py, value)?)?;
}
let row = row.unbind();
fresh.insert(owner, row.clone_ref(py));
Ok(row)
}

View file

@ -109,9 +109,7 @@ def test_evaluation_and_decisions_rows_with_zero_or_missing_output_price_pass(mo
def test_chat_rows_with_output_price_pass() -> None:
head: Final = _snapshot(
{**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)}
)
head: Final = _snapshot({**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)})
assert _failures(head) == ()
@ -186,32 +184,50 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
)
_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it"
_REMOVED_VISION: Final = (
"openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it"
)
_REMOVED_B: Final = "openrouter/b is removed before a past deprecation_date; set deprecation_date first"
_PAST: Final = "2000-01-01"
_FUTURE: Final = "2999-01-01"
def test_bot_may_not_remove_models() -> None:
def _base_with(**rows: object) -> object:
return _snapshot({**BASE_MAP, **rows})
def test_a_human_pr_may_not_remove_a_model_without_a_past_deprecation_date() -> None:
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
assert _failures(head, bot=False) == (_REMOVED_B,)
assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b")
@pytest.mark.parametrize(
("deprecation_date", "expected"),
[(_PAST, ()), (_FUTURE, (_REMOVED_B,))],
ids=["past", "future"],
)
def test_a_model_is_removable_once_its_base_deprecation_date_has_passed(
deprecation_date: str, expected: tuple[str, ...]
) -> None:
base = _base_with(**{"openrouter/b": _entry(2e-06, deprecation_date=deprecation_date)})
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
failures = guard.guard_failures(base, head, MAP_FILES, False, evaluation_output_price_sources={})
assert failures == expected
def test_a_null_row_is_refused_since_released_wheels_crash_on_it() -> None:
head = _snapshot({**BASE_MAP, "openrouter/b": None})
assert any("openrouter/b" in failure for failure in _failures(head, bot=False))
@pytest.mark.parametrize("field", ["supports_vision", "max_tokens", "input_cost_per_token"])
def test_a_human_pr_may_remove_any_field(field: str) -> None:
head = _snapshot({**BASE_MAP, "openrouter/a": {k: v for k, v in BASE_MAP["openrouter/a"].items() if k != field}})
assert _failures(head, bot=False) == ()
def test_bot_may_not_remove_fields() -> None:
head = _snapshot({**BASE_MAP, "openrouter/a": _entry()})
assert _failures(head, bot=False) == (_REMOVED_VISION,)
assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision")
def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None:
head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None})
assert _failures(head, bot=False) == ()
def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None:
head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}})
assert _failures(head, bot=False) == ()
assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",)
def test_bot_may_not_change_special_root_keys() -> None: