mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
wip
This commit is contained in:
parent
d5155204cd
commit
61c28232db
8 changed files with 923 additions and 363 deletions
|
|
@ -564,51 +564,4 @@ mod tests {
|
|||
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
|
||||
);
|
||||
}
|
||||
use serde_json::json;
|
||||
|
||||
fn parse(document: Value) -> Loaded<Catalog> {
|
||||
Catalog::parse(document.to_string().as_bytes()).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_rejected_row_is_replaced_by_bundled_or_left_out() {
|
||||
let bundled =
|
||||
parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}}))
|
||||
.value;
|
||||
let fetched = parse(json!({
|
||||
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1},
|
||||
"acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"},
|
||||
"kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0}
|
||||
}));
|
||||
|
||||
let (catalog, rejections) = fetched
|
||||
.value
|
||||
.with_rejected_replaced(&fetched.errors, &bundled);
|
||||
|
||||
let replaced: Vec<(&str, bool)> = rejections
|
||||
.iter()
|
||||
.map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled))
|
||||
.collect();
|
||||
assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]);
|
||||
assert_eq!(
|
||||
catalog
|
||||
.lookup("gpt-4o")
|
||||
.unwrap()
|
||||
.entry
|
||||
.field("input_cost_per_token"),
|
||||
Some(&json!(1.0))
|
||||
);
|
||||
assert_eq!(catalog.lookup("GPT-4O").unwrap().matched_key, "gpt-4o");
|
||||
assert!(catalog.lookup("acme/new").is_none());
|
||||
assert!(catalog.lookup("ACME/NEW").is_none());
|
||||
assert!(catalog.lookup("kept").is_some());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_null_row_is_a_removal_not_an_error() {
|
||||
let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
|
||||
assert!(loaded.errors.is_empty(), "{:?}", loaded.errors);
|
||||
assert_eq!(loaded.value.removals().collect::<Vec<_>>(), ["retired"]);
|
||||
assert_eq!(loaded.value.served().count(), 1);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
271
litellm-rust/crates/model-catalog/src/install.rs
Normal file
271
litellm-rust/crates/model-catalog/src/install.rs
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::catalog::Catalog;
|
||||
use crate::layers::Layers;
|
||||
use crate::wire::Fields;
|
||||
|
||||
/// What the bound dict holds for one field, as litellm compares it
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Held {
|
||||
Absent,
|
||||
Json(Value),
|
||||
/// A value that does not convert to JSON, which only the user can have written
|
||||
Foreign,
|
||||
}
|
||||
|
||||
/// What the bound dict holds under one key
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Slot {
|
||||
Missing,
|
||||
/// Anything but a dict, which only the user can have written
|
||||
NotARow,
|
||||
Row,
|
||||
}
|
||||
|
||||
/// The dict install writes into, `litellm.model_cost` in the Python host
|
||||
pub trait BoundMap {
|
||||
type Error;
|
||||
|
||||
fn slot(&self, key: &str) -> Result<Slot, Self::Error>;
|
||||
/// Called only for a key whose slot is `Row`
|
||||
fn held(&self, key: &str, field: &str) -> Result<Held, Self::Error>;
|
||||
/// The row's string-named fields, called only for a key whose slot is `Row`
|
||||
fn fields(&self, key: &str) -> Result<Vec<(String, Held)>, Self::Error>;
|
||||
fn row_is_empty(&self, key: &str) -> Result<bool, Self::Error>;
|
||||
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Self::Error>;
|
||||
fn del_field(&mut self, key: &str, field: &str) -> Result<(), Self::Error>;
|
||||
/// A new row under `key`, the same object for every key given the same `group` in one install
|
||||
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Self::Error>;
|
||||
fn del_key(&mut self, key: &str) -> Result<(), Self::Error>;
|
||||
/// Deletes every key, string or not, that `keep` refuses, and returns how many
|
||||
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result<usize, Self::Error>;
|
||||
}
|
||||
|
||||
/// The field values litellm last wrote into one dict object
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Record {
|
||||
dict_id: usize,
|
||||
rows: HashMap<String, Fields>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Outcome {
|
||||
pub record: Record,
|
||||
/// Keys set because the row differed or was new
|
||||
pub written: usize,
|
||||
/// Keys deleted because the map no longer serves them
|
||||
pub removed: usize,
|
||||
}
|
||||
|
||||
/// The layers and what the last install wrote, kept between installs
|
||||
#[derive(Debug)]
|
||||
pub struct Installs {
|
||||
layers: Layers,
|
||||
record: Option<Record>,
|
||||
}
|
||||
|
||||
/// One install between `Installs::begin` and `Installs::finish`
|
||||
#[derive(Debug)]
|
||||
pub struct Pending {
|
||||
layers: Layers,
|
||||
previous: Option<Record>,
|
||||
dict_id: usize,
|
||||
}
|
||||
|
||||
impl Installs {
|
||||
pub fn new(bundled: Arc<Catalog>) -> Self {
|
||||
Self {
|
||||
layers: Layers::new(bundled),
|
||||
record: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces the fetched layer. The last record applies only while `dict_id` names the dict it was written into
|
||||
pub fn begin(&mut self, fetched: Arc<Catalog>, dict_id: usize) -> Pending {
|
||||
self.layers.replace_fetched(fetched);
|
||||
Pending {
|
||||
layers: self.layers.clone(),
|
||||
previous: self
|
||||
.record
|
||||
.clone()
|
||||
.filter(|record| record.dict_id == dict_id),
|
||||
dict_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(&mut self, record: Record) {
|
||||
self.record = Some(record);
|
||||
}
|
||||
}
|
||||
|
||||
impl Pending {
|
||||
pub fn compose(&self) -> IndexMap<String, Fields> {
|
||||
self.layers.compose()
|
||||
}
|
||||
|
||||
/// A field is written only while litellm still owns it, meaning the dict holds the value last written there, or
|
||||
/// nothing where nothing was written. Without a record for this dict the dict is written in full, which is main's
|
||||
/// result after rebind then adopt
|
||||
pub fn write<M: BoundMap>(
|
||||
self,
|
||||
map: &mut M,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
) -> Result<Outcome, M::Error> {
|
||||
match self.previous {
|
||||
Some(previous) => write_owned(map, composed, &self.layers, previous),
|
||||
None => write_in_full(map, composed, &self.layers, self.dict_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_owned<M: BoundMap>(
|
||||
map: &mut M,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
previous: Record,
|
||||
) -> Result<Outcome, M::Error> {
|
||||
let mut rows = previous.rows;
|
||||
let mut written = 0;
|
||||
let mut removed = 0;
|
||||
let keys: Vec<String> = composed
|
||||
.keys()
|
||||
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
|
||||
.cloned()
|
||||
.collect();
|
||||
for key in keys {
|
||||
let target = composed.get(&key);
|
||||
let last = rows.remove(&key);
|
||||
match map.slot(&key)? {
|
||||
Slot::Missing => match (target, last) {
|
||||
(Some(target), None) => {
|
||||
map.insert_row(&key, group(layers, &key), target)?;
|
||||
rows.insert(key, target.clone());
|
||||
written += 1;
|
||||
}
|
||||
(Some(_), Some(last)) => {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
(None, _) => {}
|
||||
},
|
||||
Slot::NotARow => {
|
||||
if let Some(last) = last {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
}
|
||||
Slot::Row => {
|
||||
let last = last.unwrap_or_default();
|
||||
let (owned, changed) = write_owned_fields(map, &key, target, &last)?;
|
||||
if target.is_none() && changed && map.row_is_empty(&key)? {
|
||||
map.del_key(&key)?;
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
written += usize::from(changed);
|
||||
if !owned.is_empty() {
|
||||
rows.insert(key, owned);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record {
|
||||
dict_id: previous.dict_id,
|
||||
rows,
|
||||
},
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
fn write_in_full<M: BoundMap>(
|
||||
map: &mut M,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
dict_id: usize,
|
||||
) -> Result<Outcome, M::Error> {
|
||||
let removed = map.retain_keys(&|key| composed.contains_key(key))?;
|
||||
let mut written = 0;
|
||||
for (key, target) in composed {
|
||||
if map.slot(key)? != Slot::Row {
|
||||
map.insert_row(key, group(layers, key), target)?;
|
||||
written += 1;
|
||||
continue;
|
||||
}
|
||||
let mut current = Fields::new();
|
||||
for (field, held) in map.fields(key)? {
|
||||
match held {
|
||||
Held::Json(value) => {
|
||||
current.insert(field, value);
|
||||
}
|
||||
Held::Absent | Held::Foreign => map.del_field(key, &field)?,
|
||||
}
|
||||
}
|
||||
let (_, changed) = write_owned_fields(map, key, Some(target), ¤t)?;
|
||||
written += usize::from(changed);
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record {
|
||||
dict_id,
|
||||
rows: composed
|
||||
.iter()
|
||||
.map(|(key, fields)| (key.clone(), fields.clone()))
|
||||
.collect(),
|
||||
},
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
|
||||
fn write_owned_fields<M: BoundMap>(
|
||||
map: &mut M,
|
||||
key: &str,
|
||||
target: Option<&Fields>,
|
||||
last: &Fields,
|
||||
) -> Result<(Fields, bool), M::Error> {
|
||||
let empty = Fields::new();
|
||||
let target = target.unwrap_or(&empty);
|
||||
let fields: Vec<&String> = target
|
||||
.keys()
|
||||
.chain(last.keys().filter(|field| !target.contains_key(*field)))
|
||||
.collect();
|
||||
let mut owned = Fields::new();
|
||||
let mut changed = false;
|
||||
for field in fields {
|
||||
let held = map.held(key, field)?;
|
||||
let litellm_owns = match (&held, last.get(field)) {
|
||||
(Held::Absent, None) => true,
|
||||
(Held::Json(value), Some(last)) => value == last,
|
||||
_ => false,
|
||||
};
|
||||
if !litellm_owns {
|
||||
if let Some(last) = last.get(field) {
|
||||
owned.insert(field.clone(), last.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match target.get(field) {
|
||||
Some(value) => {
|
||||
if held != Held::Json(value.clone()) {
|
||||
map.set_field(key, field, value)?;
|
||||
changed = true;
|
||||
}
|
||||
owned.insert(field.clone(), value.clone());
|
||||
}
|
||||
None if held != Held::Absent => {
|
||||
map.del_field(key, field)?;
|
||||
changed = true;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
Ok((owned, changed))
|
||||
}
|
||||
|
||||
fn group<'a>(layers: &'a Layers, key: &'a str) -> &'a str {
|
||||
layers.alias_owner(key).unwrap_or(key)
|
||||
}
|
||||
|
|
@ -101,63 +101,3 @@ fn without_aliases(fields: &Fields) -> Fields {
|
|||
.map(|(field, value)| (field.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn catalog(document: Value) -> Arc<Catalog> {
|
||||
Arc::new(
|
||||
Catalog::parse(document.to_string().as_bytes())
|
||||
.unwrap()
|
||||
.value,
|
||||
)
|
||||
}
|
||||
|
||||
fn bundled() -> Arc<Catalog> {
|
||||
catalog(json!({
|
||||
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
|
||||
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0}
|
||||
}))
|
||||
}
|
||||
|
||||
fn composed(fetched: Value) -> IndexMap<String, Fields> {
|
||||
let mut layers = Layers::new(bundled());
|
||||
layers.replace_fetched(catalog(fetched));
|
||||
layers.compose()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_field_the_fetched_row_lacks_is_served_from_bundled() {
|
||||
let rows = composed(
|
||||
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}),
|
||||
);
|
||||
assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0));
|
||||
assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_null_field_removes_it() {
|
||||
let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}));
|
||||
assert!(!rows["gpt-4o"].contains_key("max_tokens"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() {
|
||||
let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}}));
|
||||
let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
|
||||
assert!(kept.contains_key("retired"));
|
||||
assert!(!retired.contains_key("retired"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn aliases_are_served_under_every_name_without_the_aliases_field() {
|
||||
let rows =
|
||||
composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}}));
|
||||
assert_eq!(rows["new-alias"], rows["new"]);
|
||||
assert!(!rows["new"].contains_key("aliases"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ mod bundled;
|
|||
mod catalog;
|
||||
mod error;
|
||||
mod fallback;
|
||||
mod install;
|
||||
mod layers;
|
||||
mod lower;
|
||||
mod model;
|
||||
|
|
@ -53,6 +54,7 @@ pub use bundled::*;
|
|||
pub use catalog::*;
|
||||
pub use error::*;
|
||||
pub use fallback::*;
|
||||
pub use install::*;
|
||||
pub use layers::*;
|
||||
pub use model::*;
|
||||
pub use model_info::*;
|
||||
|
|
|
|||
511
litellm-rust/crates/model-catalog/tests/policy.rs
Normal file
511
litellm-rust/crates/model-catalog/tests/policy.rs
Normal file
|
|
@ -0,0 +1,511 @@
|
|||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{
|
||||
BoundMap, Catalog, Error, Held, Installs, IntegrityLimits, Outcome, Pending, Slot,
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const FOREIGN: &str = "<foreign>";
|
||||
const NOT_A_ROW: &str = "<not a row>";
|
||||
|
||||
fn bundled_document() -> Value {
|
||||
json!({
|
||||
"sample_spec": {"litellm_provider": "one of the providers", "input_cost_per_token": 0.0},
|
||||
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
|
||||
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0},
|
||||
"claude": {"litellm_provider": "anthropic", "input_cost_per_token": 3.0, "aliases": ["claude-latest"]}
|
||||
})
|
||||
}
|
||||
|
||||
fn parse(document: &Value) -> Catalog {
|
||||
Catalog::parse(document.to_string().as_bytes())
|
||||
.unwrap()
|
||||
.value
|
||||
}
|
||||
|
||||
/// What python-bridge's `parse` hands to install for a fetched body
|
||||
fn fetched(document: &Value, bundled: &Catalog) -> Arc<Catalog> {
|
||||
let loaded = Catalog::parse(document.to_string().as_bytes()).unwrap();
|
||||
Arc::new(
|
||||
loaded
|
||||
.value
|
||||
.with_rejected_replaced(&loaded.errors, bundled)
|
||||
.0,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Stored {
|
||||
Json(Value),
|
||||
Foreign,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum Entry {
|
||||
Row(usize),
|
||||
NotARow,
|
||||
}
|
||||
|
||||
/// A Python dict of rows, where two keys can hold the same row object
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct Dict {
|
||||
keys: IndexMap<String, Entry>,
|
||||
rows: Vec<IndexMap<String, Stored>>,
|
||||
}
|
||||
|
||||
struct Session<'a> {
|
||||
dict: &'a mut Dict,
|
||||
groups: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
impl Dict {
|
||||
fn row_mut(&mut self, key: &str) -> &mut IndexMap<String, Stored> {
|
||||
match self.keys[key] {
|
||||
Entry::Row(index) => &mut self.rows[index],
|
||||
Entry::NotARow => panic!("{key} is not a row"),
|
||||
}
|
||||
}
|
||||
|
||||
fn view(&self, key: &str) -> Value {
|
||||
match self.keys.get(key) {
|
||||
None => Value::Null,
|
||||
Some(Entry::NotARow) => json!(NOT_A_ROW),
|
||||
Some(Entry::Row(index)) => self.rows[*index]
|
||||
.iter()
|
||||
.map(|(field, stored)| {
|
||||
let value = match stored {
|
||||
Stored::Json(value) => value.clone(),
|
||||
Stored::Foreign => json!(FOREIGN),
|
||||
};
|
||||
(field.clone(), value)
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn shares_row(&self, a: &str, b: &str) -> bool {
|
||||
matches!(
|
||||
(&self.keys[a], &self.keys[b]),
|
||||
(Entry::Row(a), Entry::Row(b)) if a == b
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl BoundMap for Session<'_> {
|
||||
type Error = Infallible;
|
||||
|
||||
fn slot(&self, key: &str) -> Result<Slot, Infallible> {
|
||||
Ok(match self.dict.keys.get(key) {
|
||||
None => Slot::Missing,
|
||||
Some(Entry::NotARow) => Slot::NotARow,
|
||||
Some(Entry::Row(_)) => Slot::Row,
|
||||
})
|
||||
}
|
||||
|
||||
fn held(&self, key: &str, field: &str) -> Result<Held, Infallible> {
|
||||
let Entry::Row(index) = self.dict.keys[key] else {
|
||||
panic!("{key} is not a row");
|
||||
};
|
||||
Ok(match self.dict.rows[index].get(field) {
|
||||
None => Held::Absent,
|
||||
Some(Stored::Json(value)) => Held::Json(value.clone()),
|
||||
Some(Stored::Foreign) => Held::Foreign,
|
||||
})
|
||||
}
|
||||
|
||||
fn fields(&self, key: &str) -> Result<Vec<(String, Held)>, Infallible> {
|
||||
let Entry::Row(index) = self.dict.keys[key] else {
|
||||
panic!("{key} is not a row");
|
||||
};
|
||||
Ok(self.dict.rows[index]
|
||||
.iter()
|
||||
.map(|(field, stored)| {
|
||||
let held = match stored {
|
||||
Stored::Json(value) => Held::Json(value.clone()),
|
||||
Stored::Foreign => Held::Foreign,
|
||||
};
|
||||
(field.clone(), held)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn row_is_empty(&self, key: &str) -> Result<bool, Infallible> {
|
||||
Ok(matches!(self.dict.keys[key], Entry::Row(index) if self.dict.rows[index].is_empty()))
|
||||
}
|
||||
|
||||
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> Result<(), Infallible> {
|
||||
self.dict
|
||||
.row_mut(key)
|
||||
.insert(field.to_owned(), Stored::Json(value.clone()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn del_field(&mut self, key: &str, field: &str) -> Result<(), Infallible> {
|
||||
self.dict.row_mut(key).shift_remove(field);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> Result<(), Infallible> {
|
||||
let index = match self.groups.get(group) {
|
||||
Some(index) => *index,
|
||||
None => {
|
||||
self.dict.rows.push(
|
||||
fields
|
||||
.iter()
|
||||
.map(|(field, value)| (field.clone(), Stored::Json(value.clone())))
|
||||
.collect(),
|
||||
);
|
||||
let index = self.dict.rows.len() - 1;
|
||||
self.groups.insert(group.to_owned(), index);
|
||||
index
|
||||
}
|
||||
};
|
||||
self.dict.keys.insert(key.to_owned(), Entry::Row(index));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn del_key(&mut self, key: &str) -> Result<(), Infallible> {
|
||||
self.dict.keys.shift_remove(key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> Result<usize, Infallible> {
|
||||
let before = self.dict.keys.len();
|
||||
self.dict.keys.retain(|key, _| keep(key));
|
||||
Ok(before - self.dict.keys.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// What the user does to `litellm.model_cost` between two installs
|
||||
#[derive(Clone, Debug)]
|
||||
enum Edit {
|
||||
Set(&'static str, &'static str, Value),
|
||||
Unset(&'static str, &'static str),
|
||||
SetForeign(&'static str, &'static str),
|
||||
Pop(&'static str),
|
||||
Insert(&'static str, Value),
|
||||
ReplaceWithNonRow(&'static str),
|
||||
/// `litellm.model_cost = dict(litellm.model_cost)`, the copy landing at `dict_id`
|
||||
RebindToCopy {
|
||||
dict_id: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// One Python process: the bundled map installed at import, then the user and later installs
|
||||
struct World {
|
||||
bundled: Arc<Catalog>,
|
||||
installs: Installs,
|
||||
dict: Dict,
|
||||
dict_id: usize,
|
||||
}
|
||||
|
||||
impl World {
|
||||
fn new() -> Self {
|
||||
let bundled = Arc::new(parse(&bundled_document()));
|
||||
let mut world = Self {
|
||||
installs: Installs::new(Arc::clone(&bundled)),
|
||||
bundled: Arc::clone(&bundled),
|
||||
dict: Dict::default(),
|
||||
dict_id: 1,
|
||||
};
|
||||
world.install_catalog(bundled);
|
||||
world
|
||||
}
|
||||
|
||||
fn begin(&mut self, document: &Value) -> Pending {
|
||||
let catalog = fetched(document, &self.bundled);
|
||||
self.installs.begin(catalog, self.dict_id)
|
||||
}
|
||||
|
||||
fn write(&mut self, pending: Pending) -> Outcome {
|
||||
let composed = pending.compose();
|
||||
let mut session = Session {
|
||||
dict: &mut self.dict,
|
||||
groups: HashMap::new(),
|
||||
};
|
||||
let Ok(outcome) = pending.write(&mut session, &composed);
|
||||
outcome
|
||||
}
|
||||
|
||||
fn install_catalog(&mut self, catalog: Arc<Catalog>) {
|
||||
let pending = self.installs.begin(catalog, self.dict_id);
|
||||
let outcome = self.write(pending);
|
||||
self.installs.finish(outcome.record);
|
||||
}
|
||||
|
||||
fn install(&mut self, document: &Value) {
|
||||
let catalog = fetched(document, &self.bundled);
|
||||
self.install_catalog(catalog);
|
||||
}
|
||||
|
||||
fn edit(&mut self, edit: &Edit) {
|
||||
match edit {
|
||||
Edit::Set(key, field, value) => {
|
||||
self.dict
|
||||
.row_mut(key)
|
||||
.insert((*field).to_owned(), Stored::Json(value.clone()));
|
||||
}
|
||||
Edit::Unset(key, field) => {
|
||||
self.dict.row_mut(key).shift_remove(*field);
|
||||
}
|
||||
Edit::SetForeign(key, field) => {
|
||||
self.dict
|
||||
.row_mut(key)
|
||||
.insert((*field).to_owned(), Stored::Foreign);
|
||||
}
|
||||
Edit::Pop(key) => {
|
||||
self.dict.keys.shift_remove(*key);
|
||||
}
|
||||
Edit::Insert(key, row) => {
|
||||
self.dict.rows.push(
|
||||
row.as_object()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(field, value)| (field.clone(), Stored::Json(value.clone())))
|
||||
.collect(),
|
||||
);
|
||||
let index = self.dict.rows.len() - 1;
|
||||
self.dict.keys.insert((*key).to_owned(), Entry::Row(index));
|
||||
}
|
||||
Edit::ReplaceWithNonRow(key) => {
|
||||
self.dict.keys.insert((*key).to_owned(), Entry::NotARow);
|
||||
}
|
||||
Edit::RebindToCopy { dict_id } => {
|
||||
self.dict = self.dict.clone();
|
||||
self.dict_id = *dict_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn row(&self, key: &str) -> Value {
|
||||
self.dict.view(key)
|
||||
}
|
||||
}
|
||||
|
||||
fn gpt_4o(price: f64) -> Value {
|
||||
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price}})
|
||||
}
|
||||
|
||||
fn gpt_4o_row(price: f64, max_tokens: u32) -> Value {
|
||||
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": price, "max_tokens": max_tokens}})
|
||||
}
|
||||
|
||||
/// The behavior users expect. A fetched map replaces bundled at row level, bundled is only the fallback for a bad
|
||||
/// row or a bad document, and the user's own writes survive every install. Cases the code does not meet yet fail
|
||||
mod expected {
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::fetched_row_is_served_as_written(
|
||||
gpt_4o(2.0),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0})
|
||||
)]
|
||||
#[case::null_field_is_not_served(
|
||||
json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai"})
|
||||
)]
|
||||
#[case::key_the_fetched_map_leaves_out_is_not_served(gpt_4o(2.0), "retired", Value::Null)]
|
||||
#[case::new_key_is_served(
|
||||
json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}),
|
||||
"new",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 5.0})
|
||||
)]
|
||||
#[case::alias_is_served_without_the_aliases_field(
|
||||
json!({"claude": {"litellm_provider": "anthropic", "input_cost_per_token": 4.0, "aliases": ["claude-latest"]}}),
|
||||
"claude-latest",
|
||||
json!({"litellm_provider": "anthropic", "input_cost_per_token": 4.0})
|
||||
)]
|
||||
#[case::sample_spec_is_never_served(
|
||||
json!({"sample_spec": {"input_cost_per_token": 0.0}, "gpt-4o": {"litellm_provider": "openai"}}),
|
||||
"sample_spec",
|
||||
Value::Null
|
||||
)]
|
||||
#[case::rejected_row_is_served_from_bundled(
|
||||
gpt_4o(-1.0),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384})
|
||||
)]
|
||||
#[case::rejected_row_with_no_bundled_row_is_left_out(
|
||||
json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": "free"}}),
|
||||
"internal/model",
|
||||
Value::Null
|
||||
)]
|
||||
#[case::custom_map_serves_only_its_own_rows(
|
||||
json!({"internal/model": {"litellm_provider": "openai", "input_cost_per_token": 5.0}}),
|
||||
"gpt-4o",
|
||||
Value::Null
|
||||
)]
|
||||
fn layering(#[case] fetched: Value, #[case] key: &str, #[case] expected: Value) {
|
||||
let mut world = World::new();
|
||||
world.install(&fetched);
|
||||
assert_eq!(world.row(key), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn an_alias_and_its_owner_share_one_row_object() {
|
||||
let mut world = World::new();
|
||||
world.install(
|
||||
&json!({"claude": {"litellm_provider": "anthropic", "aliases": ["claude-latest"]}}),
|
||||
);
|
||||
assert!(world.dict.shares_row("claude", "claude-latest"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_documents_own_rows_are_what_install_serves() {
|
||||
let bundled = Arc::new(parse(&bundled_document()));
|
||||
let document = fetched(&gpt_4o(2.0), &bundled);
|
||||
let mut installs = Installs::new(bundled);
|
||||
let composed = installs.begin(Arc::clone(&document), 1).compose();
|
||||
let own: IndexMap<&str, &Fields> = document.served().collect();
|
||||
let served: IndexMap<&str, &Fields> = composed
|
||||
.iter()
|
||||
.map(|(key, fields)| (key.as_str(), fields))
|
||||
.collect();
|
||||
assert_eq!(own, served);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::below_the_minimum_model_count(10, 0.0, Some("below_minimum"))]
|
||||
#[case::shrunk_against_the_bundled_count(1, 0.9, Some("shrunk"))]
|
||||
#[case::within_both_limits(1, 0.3, None)]
|
||||
fn whole_map_checks(
|
||||
#[case] min_model_count: usize,
|
||||
#[case] max_shrink_ratio: f64,
|
||||
#[case] refused: Option<&str>,
|
||||
) {
|
||||
let bundled = parse(&bundled_document());
|
||||
let kind = parse(&gpt_4o(2.0))
|
||||
.check_size(
|
||||
bundled.model_count(),
|
||||
IntegrityLimits::new(min_model_count, max_shrink_ratio).unwrap(),
|
||||
)
|
||||
.err()
|
||||
.map(|located| match located.error {
|
||||
Error::BelowMinimum { .. } => "below_minimum",
|
||||
Error::Shrunk { .. } => "shrunk",
|
||||
other => panic!("unexpected {other}"),
|
||||
});
|
||||
assert_eq!(kind, refused);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::user_value_is_kept_and_untouched_fields_follow(
|
||||
Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)),
|
||||
gpt_4o_row(2.0, 32000),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 9.0, "max_tokens": 32000})
|
||||
)]
|
||||
#[case::user_deleted_field_stays_deleted(
|
||||
Edit::Unset("gpt-4o", "max_tokens"),
|
||||
gpt_4o_row(1.0, 32000),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 1.0})
|
||||
)]
|
||||
#[case::user_deleted_key_is_not_added_back(
|
||||
Edit::Pop("gpt-4o"),
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"gpt-4o",
|
||||
Value::Null
|
||||
)]
|
||||
#[case::user_replaced_row_keeps_only_the_users_fields(
|
||||
Edit::Insert("gpt-4o", json!({"input_cost_per_token": 9.0})),
|
||||
gpt_4o_row(2.0, 32000),
|
||||
"gpt-4o",
|
||||
json!({"input_cost_per_token": 9.0})
|
||||
)]
|
||||
#[case::user_key_no_layer_has_is_untouched(
|
||||
Edit::Insert("mine", json!({"input_cost_per_token": 9.0})),
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"mine",
|
||||
json!({"input_cost_per_token": 9.0})
|
||||
)]
|
||||
#[case::user_key_a_later_fetch_brings_keeps_the_users_fields(
|
||||
Edit::Insert("new", json!({"input_cost_per_token": 9.0})),
|
||||
json!({"new": {"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 5}}),
|
||||
"new",
|
||||
json!({"input_cost_per_token": 9.0, "litellm_provider": "openai", "max_tokens": 5})
|
||||
)]
|
||||
#[case::key_the_fetched_map_drops_keeps_only_the_users_fields(
|
||||
Edit::Set("retired", "note", json!("mine")),
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"retired",
|
||||
json!({"note": "mine"})
|
||||
)]
|
||||
#[case::foreign_value_is_kept(
|
||||
Edit::SetForeign("gpt-4o", "input_cost_per_token"),
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": FOREIGN, "max_tokens": 16384})
|
||||
)]
|
||||
#[case::non_row_value_is_kept(Edit::ReplaceWithNonRow("gpt-4o"), gpt_4o_row(2.0, 16384), "gpt-4o", json!(NOT_A_ROW))]
|
||||
#[case::user_write_equal_to_litellms_value_follows_later_fetches(
|
||||
Edit::Set("gpt-4o", "input_cost_per_token", json!(1.0)),
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384})
|
||||
)]
|
||||
#[case::rebind_is_written_in_full(
|
||||
Edit::RebindToCopy { dict_id: 2 },
|
||||
gpt_4o_row(2.0, 16384),
|
||||
"gpt-4o",
|
||||
json!({"litellm_provider": "openai", "input_cost_per_token": 2.0, "max_tokens": 16384})
|
||||
)]
|
||||
fn direct_writes(
|
||||
#[case] edit: Edit,
|
||||
#[case] fetched: Value,
|
||||
#[case] key: &str,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let mut world = World::new();
|
||||
world.edit(&edit);
|
||||
world.install(&fetched);
|
||||
assert_eq!(world.row(key), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn rebind_drops_direct_writes() {
|
||||
let mut world = World::new();
|
||||
world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)));
|
||||
world.edit(&Edit::Insert("mine", json!({"input_cost_per_token": 9.0})));
|
||||
world.edit(&Edit::RebindToCopy { dict_id: 2 });
|
||||
world.install(&gpt_4o(2.0));
|
||||
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(2.0));
|
||||
assert_eq!(world.row("mine"), Value::Null);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn overlapping_installs_serve_the_last_one_begun_and_keep_following_fetches() {
|
||||
let mut world = World::new();
|
||||
let first = world.begin(&gpt_4o(2.0));
|
||||
let second = world.begin(&gpt_4o(3.0));
|
||||
let first_outcome = world.write(first);
|
||||
world.installs.finish(first_outcome.record);
|
||||
let second_outcome = world.write(second);
|
||||
world.installs.finish(second_outcome.record);
|
||||
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(3.0));
|
||||
|
||||
world.install(&gpt_4o(4.0));
|
||||
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(4.0));
|
||||
}
|
||||
}
|
||||
|
||||
/// Not decided yet. Each case pins what the code does today, so deciding one changes its expected value
|
||||
mod open {
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
fn a_rebind_to_a_dict_at_the_old_address_keeps_direct_writes() {
|
||||
let mut world = World::new();
|
||||
world.edit(&Edit::Set("gpt-4o", "input_cost_per_token", json!(9.0)));
|
||||
world.edit(&Edit::RebindToCopy { dict_id: 1 });
|
||||
world.install(&gpt_4o(2.0));
|
||||
assert_eq!(world.row("gpt-4o")["input_cost_per_token"], json!(9.0));
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ use std::sync::{Arc, LazyLock};
|
|||
|
||||
use litellm_host_python::{release_gil, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Installs, IntegrityLimits};
|
||||
use parking_lot::Mutex;
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
use pyo3::prelude::*;
|
||||
|
|
@ -16,7 +16,7 @@ use pyo3::types::{PyBytes, PyDict, PyModule};
|
|||
use serde::Serialize;
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
use install::{Record, write_into};
|
||||
use install::BoundDict;
|
||||
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
|
||||
|
||||
static BUNDLED: LazyLock<Arc<Contents>> = LazyLock::new(|| {
|
||||
|
|
@ -72,16 +72,11 @@ pub(crate) struct Unavailable {
|
|||
reason: String,
|
||||
}
|
||||
|
||||
struct State {
|
||||
layers: Layers,
|
||||
record: Option<Record>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct CatalogHandle {
|
||||
module: Py<PyModule>,
|
||||
attr: String,
|
||||
state: Mutex<State>,
|
||||
installs: Mutex<Installs>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
|
|
@ -126,10 +121,7 @@ impl CatalogHandle {
|
|||
Self {
|
||||
module: module.unbind(),
|
||||
attr,
|
||||
state: Mutex::new(State {
|
||||
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
|
||||
record: None,
|
||||
}),
|
||||
installs: Mutex::new(Installs::new(Arc::clone(&BUNDLED.catalog))),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -177,20 +169,13 @@ impl CatalogHandle {
|
|||
.cast::<PyDict>()
|
||||
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?;
|
||||
let dict_id = dict.as_ptr() as usize;
|
||||
let (layers, record) = {
|
||||
let mut state = self.state.lock();
|
||||
state
|
||||
.layers
|
||||
.replace_fetched(Arc::clone(&parsed.contents.catalog));
|
||||
let record = state
|
||||
.record
|
||||
.clone()
|
||||
.filter(|record| record.dict_id == dict_id);
|
||||
(state.layers.clone(), record)
|
||||
};
|
||||
let composed = release_gil(py, || layers.compose());
|
||||
let outcome = write_into(py, dict, &composed, &layers, record)?;
|
||||
self.state.lock().record = Some(outcome.record);
|
||||
let pending = self
|
||||
.installs
|
||||
.lock()
|
||||
.begin(Arc::clone(&parsed.contents.catalog), dict_id);
|
||||
let composed = release_gil(py, || pending.compose());
|
||||
let outcome = pending.write(&mut BoundDict::new(dict), &composed)?;
|
||||
self.installs.lock().finish(outcome.record);
|
||||
Ok(Installed {
|
||||
count: composed.len(),
|
||||
written: outcome.written,
|
||||
|
|
|
|||
|
|
@ -1,234 +1,116 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use litellm_host_python::{from_py, to_py};
|
||||
use litellm_model_catalog::Layers;
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{BoundMap, Held, Slot};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use serde_json::Value;
|
||||
|
||||
/// The field values litellm last wrote into one dict object
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Record {
|
||||
pub(crate) dict_id: usize,
|
||||
rows: HashMap<String, Fields>,
|
||||
/// The dict bound to `litellm.model_cost` for one install, under one GIL hold
|
||||
pub(crate) struct BoundDict<'a, 'py> {
|
||||
dict: &'a Bound<'py, PyDict>,
|
||||
/// The row object each alias group shares in this install, as main's alias expansion gives them
|
||||
groups: HashMap<String, Py<PyDict>>,
|
||||
}
|
||||
|
||||
pub(crate) struct Outcome {
|
||||
pub(crate) record: Record,
|
||||
pub(crate) written: usize,
|
||||
pub(crate) removed: usize,
|
||||
}
|
||||
|
||||
/// What the dict holds for one field, as litellm compares it
|
||||
#[derive(PartialEq)]
|
||||
enum Held {
|
||||
Absent,
|
||||
Json(Value),
|
||||
/// A value that does not convert to JSON, which only the user can have written
|
||||
Foreign,
|
||||
}
|
||||
|
||||
/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the
|
||||
/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the
|
||||
/// last install is written in full, which is main's result after rebind then adopt
|
||||
pub(crate) fn write_into(
|
||||
py: Python<'_>,
|
||||
dict: &Bound<'_, PyDict>,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
previous: Option<Record>,
|
||||
) -> PyResult<Outcome> {
|
||||
let dict_id = dict.as_ptr() as usize;
|
||||
let Some(previous) = previous else {
|
||||
return write_in_full(py, dict, composed, layers, dict_id);
|
||||
};
|
||||
let mut rows = previous.rows;
|
||||
let mut written = 0;
|
||||
let mut removed = 0;
|
||||
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
|
||||
let keys: Vec<String> = composed
|
||||
.keys()
|
||||
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
|
||||
.cloned()
|
||||
.collect();
|
||||
for key in keys {
|
||||
let target = composed.get(&key);
|
||||
let last = rows.remove(&key);
|
||||
let Some(current) = dict.get_item(&key)? else {
|
||||
match (target, last) {
|
||||
(Some(target), None) => {
|
||||
dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?;
|
||||
rows.insert(key, target.clone());
|
||||
written += 1;
|
||||
}
|
||||
(_, Some(last)) if target.is_some() => {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let Ok(row) = current.cast::<PyDict>() else {
|
||||
if let Some(last) = last {
|
||||
rows.insert(key, last);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let last = last.unwrap_or_default();
|
||||
let (owned, changed) = write_owned_fields(py, row, target, &last)?;
|
||||
if target.is_none() && changed && row.is_empty() {
|
||||
dict.del_item(&key)?;
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
written += usize::from(changed);
|
||||
if !owned.is_empty() {
|
||||
rows.insert(key, owned);
|
||||
impl<'a, 'py> BoundDict<'a, 'py> {
|
||||
pub(crate) fn new(dict: &'a Bound<'py, PyDict>) -> Self {
|
||||
Self {
|
||||
dict,
|
||||
groups: HashMap::new(),
|
||||
}
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record { dict_id, rows },
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
|
||||
fn py(&self) -> Python<'py> {
|
||||
self.dict.py()
|
||||
}
|
||||
|
||||
fn row(&self, key: &str) -> PyResult<Bound<'py, PyDict>> {
|
||||
let row = self
|
||||
.dict
|
||||
.get_item(key)?
|
||||
.ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(key.to_owned()))?;
|
||||
Ok(row.cast_into::<PyDict>()?)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_in_full(
|
||||
py: Python<'_>,
|
||||
dict: &Bound<'_, PyDict>,
|
||||
composed: &IndexMap<String, Fields>,
|
||||
layers: &Layers,
|
||||
dict_id: usize,
|
||||
) -> PyResult<Outcome> {
|
||||
let stale: Vec<Bound<'_, PyAny>> = dict
|
||||
.keys()
|
||||
.iter()
|
||||
.filter(|key| {
|
||||
key.extract::<String>()
|
||||
.map_or(true, |key| !composed.contains_key(&key))
|
||||
fn held_value(value: &Bound<'_, PyAny>) -> Held {
|
||||
from_py::<Value>(value).map_or(Held::Foreign, Held::Json)
|
||||
}
|
||||
|
||||
impl BoundMap for BoundDict<'_, '_> {
|
||||
type Error = PyErr;
|
||||
|
||||
fn slot(&self, key: &str) -> PyResult<Slot> {
|
||||
Ok(match self.dict.get_item(key)? {
|
||||
None => Slot::Missing,
|
||||
Some(value) if value.is_instance_of::<PyDict>() => Slot::Row,
|
||||
Some(_) => Slot::NotARow,
|
||||
})
|
||||
.collect();
|
||||
let removed = stale.len();
|
||||
for key in stale {
|
||||
dict.del_item(key)?;
|
||||
}
|
||||
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
|
||||
let mut written = 0;
|
||||
for (key, target) in composed {
|
||||
let current = dict.get_item(key)?;
|
||||
let row = current.as_ref().and_then(|row| row.cast::<PyDict>().ok());
|
||||
match row {
|
||||
Some(row) => {
|
||||
let owned_by_litellm: Fields = row
|
||||
.iter()
|
||||
.filter_map(|(field, value)| {
|
||||
let field = field.extract::<String>().ok()?;
|
||||
Some((field, from_py::<Value>(&value).ok()?))
|
||||
})
|
||||
.collect();
|
||||
let extra: Vec<String> = row
|
||||
.keys()
|
||||
.iter()
|
||||
.filter_map(|field| field.extract::<String>().ok())
|
||||
.filter(|field| !owned_by_litellm.contains_key(field))
|
||||
.collect();
|
||||
for field in extra {
|
||||
row.del_item(field)?;
|
||||
}
|
||||
let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?;
|
||||
written += usize::from(changed);
|
||||
}
|
||||
|
||||
fn held(&self, key: &str, field: &str) -> PyResult<Held> {
|
||||
Ok(self
|
||||
.row(key)?
|
||||
.get_item(field)?
|
||||
.map_or(Held::Absent, |value| held_value(&value)))
|
||||
}
|
||||
|
||||
fn fields(&self, key: &str) -> PyResult<Vec<(String, Held)>> {
|
||||
Ok(self
|
||||
.row(key)?
|
||||
.iter()
|
||||
.filter_map(|(field, value)| {
|
||||
Some((field.extract::<String>().ok()?, held_value(&value)))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn row_is_empty(&self, key: &str) -> PyResult<bool> {
|
||||
Ok(self.row(key)?.is_empty())
|
||||
}
|
||||
|
||||
fn set_field(&mut self, key: &str, field: &str, value: &Value) -> PyResult<()> {
|
||||
self.row(key)?.set_item(field, to_py(self.py(), value)?)
|
||||
}
|
||||
|
||||
fn del_field(&mut self, key: &str, field: &str) -> PyResult<()> {
|
||||
self.row(key)?.del_item(field)
|
||||
}
|
||||
|
||||
fn insert_row(&mut self, key: &str, group: &str, fields: &Fields) -> PyResult<()> {
|
||||
let py = self.py();
|
||||
let row = match self.groups.get(group) {
|
||||
Some(row) => row.clone_ref(py),
|
||||
None => {
|
||||
dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?;
|
||||
written += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Outcome {
|
||||
record: Record {
|
||||
dict_id,
|
||||
rows: composed
|
||||
.iter()
|
||||
.map(|(key, fields)| (key.clone(), fields.clone()))
|
||||
.collect(),
|
||||
},
|
||||
written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
|
||||
fn write_owned_fields(
|
||||
py: Python<'_>,
|
||||
row: &Bound<'_, PyDict>,
|
||||
target: Option<&Fields>,
|
||||
last: &Fields,
|
||||
) -> PyResult<(Fields, bool)> {
|
||||
let empty = Fields::new();
|
||||
let target = target.unwrap_or(&empty);
|
||||
let fields: Vec<&String> = target
|
||||
.keys()
|
||||
.chain(last.keys().filter(|field| !target.contains_key(*field)))
|
||||
.collect();
|
||||
let mut owned = Fields::new();
|
||||
let mut changed = false;
|
||||
for field in fields {
|
||||
let held = held(row, field)?;
|
||||
let litellm_owns = match (&held, last.get(field)) {
|
||||
(Held::Absent, None) => true,
|
||||
(Held::Json(value), Some(last)) => value == last,
|
||||
_ => false,
|
||||
};
|
||||
if !litellm_owns {
|
||||
if let Some(last) = last.get(field) {
|
||||
owned.insert(field.clone(), last.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match target.get(field) {
|
||||
Some(value) => {
|
||||
if held != Held::Json(value.clone()) {
|
||||
let row = PyDict::new(py);
|
||||
for (field, value) in fields {
|
||||
row.set_item(field, to_py(py, value)?)?;
|
||||
changed = true;
|
||||
}
|
||||
owned.insert(field.clone(), value.clone());
|
||||
let row = row.unbind();
|
||||
self.groups.insert(group.to_owned(), row.clone_ref(py));
|
||||
row
|
||||
}
|
||||
None if held != Held::Absent => {
|
||||
row.del_item(field)?;
|
||||
changed = true;
|
||||
}
|
||||
None => {}
|
||||
};
|
||||
self.dict.set_item(key, row)
|
||||
}
|
||||
|
||||
fn del_key(&mut self, key: &str) -> PyResult<()> {
|
||||
self.dict.del_item(key)
|
||||
}
|
||||
|
||||
fn retain_keys(&mut self, keep: &dyn Fn(&str) -> bool) -> PyResult<usize> {
|
||||
let stale: Vec<Bound<'_, PyAny>> = self
|
||||
.dict
|
||||
.keys()
|
||||
.iter()
|
||||
.filter(|key| key.extract::<String>().map_or(true, |key| !keep(&key)))
|
||||
.collect();
|
||||
let removed = stale.len();
|
||||
for key in stale {
|
||||
self.dict.del_item(key)?;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
Ok((owned, changed))
|
||||
}
|
||||
|
||||
fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult<Held> {
|
||||
Ok(match row.get_item(field)? {
|
||||
None => Held::Absent,
|
||||
Some(value) => from_py::<Value>(&value).map_or(Held::Foreign, Held::Json),
|
||||
})
|
||||
}
|
||||
|
||||
/// A new row, the same object for every key of one alias group, as main's alias expansion gives them
|
||||
fn shared_row(
|
||||
py: Python<'_>,
|
||||
fresh: &mut HashMap<String, Py<PyDict>>,
|
||||
layers: &Layers,
|
||||
key: &str,
|
||||
fields: &Fields,
|
||||
) -> PyResult<Py<PyDict>> {
|
||||
let owner = layers.alias_owner(key).unwrap_or(key).to_owned();
|
||||
if let Some(row) = fresh.get(&owner) {
|
||||
return Ok(row.clone_ref(py));
|
||||
}
|
||||
let row = PyDict::new(py);
|
||||
for (field, value) in fields {
|
||||
row.set_item(field, to_py(py, value)?)?;
|
||||
}
|
||||
let row = row.unbind();
|
||||
fresh.insert(owner, row.clone_ref(py));
|
||||
Ok(row)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,9 +109,7 @@ def test_evaluation_and_decisions_rows_with_zero_or_missing_output_price_pass(mo
|
|||
|
||||
|
||||
def test_chat_rows_with_output_price_pass() -> None:
|
||||
head: Final = _snapshot(
|
||||
{**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)}
|
||||
)
|
||||
head: Final = _snapshot({**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)})
|
||||
assert _failures(head) == ()
|
||||
|
||||
|
||||
|
|
@ -186,32 +184,50 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
|
|||
)
|
||||
|
||||
|
||||
_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it"
|
||||
_REMOVED_VISION: Final = (
|
||||
"openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it"
|
||||
)
|
||||
_REMOVED_B: Final = "openrouter/b is removed before a past deprecation_date; set deprecation_date first"
|
||||
_PAST: Final = "2000-01-01"
|
||||
_FUTURE: Final = "2999-01-01"
|
||||
|
||||
|
||||
def test_bot_may_not_remove_models() -> None:
|
||||
def _base_with(**rows: object) -> object:
|
||||
return _snapshot({**BASE_MAP, **rows})
|
||||
|
||||
|
||||
def test_a_human_pr_may_not_remove_a_model_without_a_past_deprecation_date() -> None:
|
||||
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
|
||||
assert _failures(head, bot=False) == (_REMOVED_B,)
|
||||
assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deprecation_date", "expected"),
|
||||
[(_PAST, ()), (_FUTURE, (_REMOVED_B,))],
|
||||
ids=["past", "future"],
|
||||
)
|
||||
def test_a_model_is_removable_once_its_base_deprecation_date_has_passed(
|
||||
deprecation_date: str, expected: tuple[str, ...]
|
||||
) -> None:
|
||||
base = _base_with(**{"openrouter/b": _entry(2e-06, deprecation_date=deprecation_date)})
|
||||
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
|
||||
failures = guard.guard_failures(base, head, MAP_FILES, False, evaluation_output_price_sources={})
|
||||
assert failures == expected
|
||||
|
||||
|
||||
def test_a_null_row_is_refused_since_released_wheels_crash_on_it() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/b": None})
|
||||
assert any("openrouter/b" in failure for failure in _failures(head, bot=False))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["supports_vision", "max_tokens", "input_cost_per_token"])
|
||||
def test_a_human_pr_may_remove_any_field(field: str) -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": {k: v for k, v in BASE_MAP["openrouter/a"].items() if k != field}})
|
||||
assert _failures(head, bot=False) == ()
|
||||
|
||||
|
||||
def test_bot_may_not_remove_fields() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": _entry()})
|
||||
assert _failures(head, bot=False) == (_REMOVED_VISION,)
|
||||
assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision")
|
||||
|
||||
|
||||
def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None})
|
||||
assert _failures(head, bot=False) == ()
|
||||
|
||||
|
||||
def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}})
|
||||
assert _failures(head, bot=False) == ()
|
||||
assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",)
|
||||
|
||||
|
||||
def test_bot_may_not_change_special_root_keys() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue