ci(model-catalog): check the cost map by loading it through model-catalog

This commit is contained in:
Devin AI 2026-10-28 22:21:53 +00:00 • committed by yujonglee
parent 015a80b0b6
commit cf2fa26737
7 changed files with 179 additions and 14 deletions

View file

@ -42,3 +42,24 @@ jobs:
HEAD_REF: ${{ github.event.pull_request.head.ref }}
run: |
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
- name: Check whether the pull request changes the cost map
id: changed
env:
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
echo "cost_map=false" >> "$GITHUB_OUTPUT"
else
echo "cost_map=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.changed.outputs.cost_map == 'true'
run: rustup toolchain install --no-self-update
- name: Load the cost map through model-catalog
if: steps.changed.outputs.cost_map == 'true'
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
-- "$RUNNER_TEMP/model_prices_and_context_window.json"

View file

@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
## Tests
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`

View file

@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
[dependencies]
litellm-cost = { workspace = true, features = ["serde"] }
litellm-error.workspace = true
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
@ -24,5 +25,4 @@ time.workspace = true
[dev-dependencies]
jsonschema = { version = "0.55.1", default-features = false }
litellm-error.workspace = true
rstest.workspace = true

View file

@ -0,0 +1,26 @@
use std::{env, fs, path::PathBuf, process::ExitCode};
use litellm_model_catalog::Catalog;
fn main() -> ExitCode {
let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
eprintln!("usage: check-registry <model_prices_and_context_window.json>");
return ExitCode::from(2);
};
let body = match fs::read(&path) {
Ok(body) => body,
Err(error) => {
eprintln!("{}: {error}", path.display());
return ExitCode::FAILURE;
}
};
let problems = Catalog::problems(&body);
for problem in &problems {
eprintln!("{}", litellm_error::Report::new(problem));
}
if problems.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
}
}

View file

@ -5,7 +5,7 @@ use indexmap::IndexMap;
use serde::Deserialize;
use serde_json::Value;
use crate::error::{Error, Loaded, Rejects, Repr};
use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
use crate::fallback::{FallbackGeneralizations, FallbackRule};
use crate::model::Model;
use crate::store::{Rows, Store};
@ -211,6 +211,36 @@ impl Catalog {
})
}
pub fn problems(body: &[u8]) -> Vec<Error> {
let Loaded {
value: catalog,
errors,
} = match Self::parse(body) {
Ok(loaded) => loaded,
Err(fatal) => return vec![fatal],
};
let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
EntryWire::read(entry.fields())
.unknown
.into_iter()
.map(move |(key, _)| {
Error::new(
pointer_child(&pointer_for_key(model), &key),
Repr::UnreadField,
)
})
});
let no_fallback_rules = catalog
.fallback_rules()
.is_none_or(<[_]>::is_empty)
.then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
errors
.into_iter()
.chain(unread)
.chain(no_fallback_rules)
.collect()
}
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
let (_, _, errors) = read_row(fields);
errors
@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
)
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
#[rstest]
#[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
#[case::not_json("{".to_owned(), vec![("", "json")])]
#[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
#[case::unread_key(
format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
vec![("/m/typo_cost", "unread_field")]
)]
#[case::empty_model_key(
format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
vec![("//future_field", "unread_field")]
)]
#[case::unread_key_on_invalid_row(
format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
)]
#[case::escaped_unread_key_on_invalid_row(
format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
)]
#[case::no_rules(
r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
#[case::empty_rules(
r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
fn problems_list_what_a_cost_map_change_must_fix(
#[case] body: String,
#[case] expected: Vec<(&str, &str)>,
) {
let problems: Vec<_> = Catalog::problems(body.as_bytes())
.iter()
.map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
.collect();
let expected: Vec<_> = expected
.into_iter()
.map(|(path, kind)| (path.to_owned(), kind))
.collect();
assert_eq!(problems, expected);
}
#[rstest]
fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
let body = br#"{
"good":{"litellm_provider":"test","custom_metadata":true},
"bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
}"#;
let loaded = Catalog::parse(body).unwrap();
assert_eq!(loaded.errors.len(), 1);
assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
assert_eq!(rejected.len(), 1);
assert!(catalog.lookup("bad").is_none());
let good = catalog.lookup("good").unwrap();
assert!(good.entry.model().is_some());
assert_eq!(
good.entry.field("custom_metadata"),
Some(&Value::Bool(true))
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
);
}
}

View file

@ -109,6 +109,12 @@ pub(crate) enum Repr {
#[source]
cause: Option<DateCause>,
},
#[error("not a field litellm reads")]
#[strum_discriminants(strum(serialize = "unread_field"))]
UnreadField,
#[error("has no rules")]
#[strum_discriminants(strum(serialize = "no_fallback_rules"))]
NoFallbackRules,
}
/// What a problem takes out of the served map
@ -148,7 +154,7 @@ impl Error {
| Repr::NoOffPeakHours
| Repr::EmptyList
| Repr::InvalidDate { .. } => Rejects::Row,
Repr::MissingProvider => Rejects::Nothing,
Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
}
}
}
@ -334,6 +340,8 @@ mod tests {
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
#[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
#[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
assert_eq!(Error::from(error).rejects(), expected);
}

View file

@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() {
}
#[rstest]
fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
let catalog = loaded.value;
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
let unrecognized: Vec<_> = catalog.unrecognized().collect();
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
assert!(
catalog
.fallback_rules()
.is_some_and(|rules| !rules.is_empty())
);
fn the_bundled_registry_has_no_problems() {
let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY)
.iter()
.map(report)
.collect();
assert!(problems.is_empty(), "{problems:#?}");
const REFERENCE_MODEL_COUNT: usize = 4303;
let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
}