mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
ci(model-catalog): check the cost map by loading it through model-catalog
This commit is contained in:
parent
015a80b0b6
commit
cf2fa26737
7 changed files with 179 additions and 14 deletions
21
.github/workflows/cost-map-guard.yml
vendored
21
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -42,3 +42,24 @@ jobs:
|
|||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
|
||||
- name: Check whether the pull request changes the cost map
|
||||
id: changed
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
|
||||
echo "cost_map=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "cost_map=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- if: steps.changed.outputs.cost_map == 'true'
|
||||
run: rustup toolchain install --no-self-update
|
||||
- name: Load the cost map through model-catalog
|
||||
if: steps.changed.outputs.cost_map == 'true'
|
||||
env:
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
|
||||
cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
|
||||
-- "$RUNNER_TEMP/model_prices_and_context_window.json"
|
||||
|
|
|
|||
|
|
@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
|
|||
## Tests
|
||||
|
||||
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
|
||||
|
||||
`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
|
|||
|
||||
[dependencies]
|
||||
litellm-cost = { workspace = true, features = ["serde"] }
|
||||
litellm-error.workspace = true
|
||||
|
||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
|
||||
|
|
@ -24,5 +25,4 @@ time.workspace = true
|
|||
|
||||
[dev-dependencies]
|
||||
jsonschema = { version = "0.55.1", default-features = false }
|
||||
litellm-error.workspace = true
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs
Normal file
26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use std::{env, fs, path::PathBuf, process::ExitCode};
|
||||
|
||||
use litellm_model_catalog::Catalog;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
|
||||
eprintln!("usage: check-registry <model_prices_and_context_window.json>");
|
||||
return ExitCode::from(2);
|
||||
};
|
||||
let body = match fs::read(&path) {
|
||||
Ok(body) => body,
|
||||
Err(error) => {
|
||||
eprintln!("{}: {error}", path.display());
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let problems = Catalog::problems(&body);
|
||||
for problem in &problems {
|
||||
eprintln!("{}", litellm_error::Report::new(problem));
|
||||
}
|
||||
if problems.is_empty() {
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ use indexmap::IndexMap;
|
|||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::error::{Error, Loaded, Rejects, Repr};
|
||||
use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
|
||||
use crate::fallback::{FallbackGeneralizations, FallbackRule};
|
||||
use crate::model::Model;
|
||||
use crate::store::{Rows, Store};
|
||||
|
|
@ -211,6 +211,36 @@ impl Catalog {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn problems(body: &[u8]) -> Vec<Error> {
|
||||
let Loaded {
|
||||
value: catalog,
|
||||
errors,
|
||||
} = match Self::parse(body) {
|
||||
Ok(loaded) => loaded,
|
||||
Err(fatal) => return vec![fatal],
|
||||
};
|
||||
let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
|
||||
EntryWire::read(entry.fields())
|
||||
.unknown
|
||||
.into_iter()
|
||||
.map(move |(key, _)| {
|
||||
Error::new(
|
||||
pointer_child(&pointer_for_key(model), &key),
|
||||
Repr::UnreadField,
|
||||
)
|
||||
})
|
||||
});
|
||||
let no_fallback_rules = catalog
|
||||
.fallback_rules()
|
||||
.is_none_or(<[_]>::is_empty)
|
||||
.then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
|
||||
errors
|
||||
.into_iter()
|
||||
.chain(unread)
|
||||
.chain(no_fallback_rules)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
|
||||
let (_, _, errors) = read_row(fields);
|
||||
errors
|
||||
|
|
@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
|
|||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
|
||||
|
||||
#[rstest]
|
||||
#[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
|
||||
#[case::not_json("{".to_owned(), vec![("", "json")])]
|
||||
#[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
|
||||
#[case::unread_key(
|
||||
format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
|
||||
vec![("/m/typo_cost", "unread_field")]
|
||||
)]
|
||||
#[case::empty_model_key(
|
||||
format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
|
||||
vec![("//future_field", "unread_field")]
|
||||
)]
|
||||
#[case::unread_key_on_invalid_row(
|
||||
format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
|
||||
vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
|
||||
)]
|
||||
#[case::escaped_unread_key_on_invalid_row(
|
||||
format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
|
||||
vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
|
||||
)]
|
||||
#[case::no_rules(
|
||||
r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
|
||||
vec![("/fallback_generalizations", "no_fallback_rules")]
|
||||
)]
|
||||
#[case::empty_rules(
|
||||
r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
|
||||
vec![("/fallback_generalizations", "no_fallback_rules")]
|
||||
)]
|
||||
fn problems_list_what_a_cost_map_change_must_fix(
|
||||
#[case] body: String,
|
||||
#[case] expected: Vec<(&str, &str)>,
|
||||
) {
|
||||
let problems: Vec<_> = Catalog::problems(body.as_bytes())
|
||||
.iter()
|
||||
.map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
|
||||
.collect();
|
||||
let expected: Vec<_> = expected
|
||||
.into_iter()
|
||||
.map(|(path, kind)| (path.to_owned(), kind))
|
||||
.collect();
|
||||
assert_eq!(problems, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
|
||||
let body = br#"{
|
||||
"good":{"litellm_provider":"test","custom_metadata":true},
|
||||
"bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
|
||||
}"#;
|
||||
let loaded = Catalog::parse(body).unwrap();
|
||||
assert_eq!(loaded.errors.len(), 1);
|
||||
assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
|
||||
let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
|
||||
assert_eq!(rejected.len(), 1);
|
||||
assert!(catalog.lookup("bad").is_none());
|
||||
let good = catalog.lookup("good").unwrap();
|
||||
assert!(good.entry.model().is_some());
|
||||
assert_eq!(
|
||||
good.entry.field("custom_metadata"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
assert!(
|
||||
Catalog::problems(body)
|
||||
.iter()
|
||||
.any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
|
||||
);
|
||||
assert!(
|
||||
Catalog::problems(body)
|
||||
.iter()
|
||||
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,6 +109,12 @@ pub(crate) enum Repr {
|
|||
#[source]
|
||||
cause: Option<DateCause>,
|
||||
},
|
||||
#[error("not a field litellm reads")]
|
||||
#[strum_discriminants(strum(serialize = "unread_field"))]
|
||||
UnreadField,
|
||||
#[error("has no rules")]
|
||||
#[strum_discriminants(strum(serialize = "no_fallback_rules"))]
|
||||
NoFallbackRules,
|
||||
}
|
||||
|
||||
/// What a problem takes out of the served map
|
||||
|
|
@ -148,7 +154,7 @@ impl Error {
|
|||
| Repr::NoOffPeakHours
|
||||
| Repr::EmptyList
|
||||
| Repr::InvalidDate { .. } => Rejects::Row,
|
||||
Repr::MissingProvider => Rejects::Nothing,
|
||||
Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -334,6 +340,8 @@ mod tests {
|
|||
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
|
||||
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
|
||||
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
|
||||
#[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
|
||||
#[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
|
||||
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
|
||||
assert_eq!(Error::from(error).rejects(), expected);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
|
||||
let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
|
||||
let catalog = loaded.value;
|
||||
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
|
||||
let unrecognized: Vec<_> = catalog.unrecognized().collect();
|
||||
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
|
||||
assert!(
|
||||
catalog
|
||||
.fallback_rules()
|
||||
.is_some_and(|rules| !rules.is_empty())
|
||||
);
|
||||
fn the_bundled_registry_has_no_problems() {
|
||||
let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY)
|
||||
.iter()
|
||||
.map(report)
|
||||
.collect();
|
||||
assert!(problems.is_empty(), "{problems:#?}");
|
||||
const REFERENCE_MODEL_COUNT: usize = 4303;
|
||||
let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
|
||||
assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue