From 7c716d5165c8d71308b4e984f232d93b364db74d Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 28 Oct 2026 12:50:51 -0700 Subject: [PATCH] refactor(model-catalog): serve the fetched map row for row The fetched map now replaces the bundled one at row level instead of patching it field by field, so a field or model the map drops is gone on every wheel, and bundled only shows through for rejected rows. The cost map guard drops the null rule, refuses null rows, and lets a PR delete a model only after its deprecation_date has passed Co-Authored-By: Claude Opus 5.5 --- ci_cd/cost_map_guard.py | 39 ++++++++-------- .../crates/model-catalog/src/catalog.rs | 9 ---- .../crates/model-catalog/src/layers.rs | 46 ++----------------- .../crates/model-catalog/tests/catalog.rs | 1 - .../test_get_model_cost_map.py | 16 ++++--- 5 files changed, 34 insertions(+), 77 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 1d95da149ad..559f95b6be0 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -6,8 +6,9 @@ skips them, since merging it keeps the base branch's copies and its head tree on branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those two files and may only add or update models. -Installed wheels layer a fetched map over their bundled one, so a key or a price or ``supports_*`` field the map -leaves out is still served from the wheel. Every pull request that drops one must write it as ``null`` instead. +Installed wheels serve the fetched map's rows as written, so a dropped field or model is gone everywhere at once. A +model is deleted only after its ``deprecation_date`` has passed at the merge base, and a row is never ``null``, +since released wheels crash on one. """ from __future__ import annotations @@ -18,6 +19,7 @@ import subprocess import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass +from datetime import date from types import MappingProxyType from typing import Final @@ -94,31 +96,30 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]: return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)} -def _retired_by_null(field: str) -> bool: - return "cost" in field or field in ("tiered_pricing", "off_peak_pricing") or field.startswith("supports_") +def _past_deprecation(entry: object, today: date) -> bool: + if not isinstance(entry, dict) or not isinstance(deprecation_date := entry.get("deprecation_date"), str): + return False + try: + return date.fromisoformat(deprecation_date) <= today + except ValueError: + return False -def _silent_removal_failures(base: Snapshot, head_map: CostMap) -> tuple[str, ...]: +def _removal_failures(base: Snapshot, head_map: CostMap) -> tuple[str, ...]: base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) if isinstance(base_map, str): return () - head_entries: Final = _entries(head_map) - removed_fields: Final = tuple( - f"{key}.{field}" - for key, entry in _entries(base_map).items() - if key in head_entries - for field in entry - if _retired_by_null(field) and field not in head_entries[key] - ) + today: Final = date.today() return ( *( - f"{key} is removed without a null; write it as null so older wheels stop serving it" - for key in base_map - if key not in head_map and key not in SPECIAL_ROOT_KEYS + f"{key} is null, which released wheels crash on. Delete it once its deprecation_date has passed" + for key, entry in head_map.items() + if entry is None ), *( - f"{ref} is removed without a null; write it as null so older wheels stop serving it" - for ref in removed_fields + f"{key} is removed before a past deprecation_date; set deprecation_date first" + for key, entry in base_map.items() + if key not in head_map and key not in SPECIAL_ROOT_KEYS and not _past_deprecation(entry, today) ), ) @@ -179,7 +180,7 @@ def guard_failures( *_file_failures( head, head_map, evaluation_output_price_sources=evaluation_output_price_sources ), - *_silent_removal_failures(base, head_map), + *_removal_failures(base, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()), ) diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 3c1844c41a1..a2feff38716 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -61,7 +61,6 @@ pub struct Catalog { store: Store, sections: Vec
, fallback_generalizations: Option, - removals: Vec, } /// Models and aliases, an alias served as its owner's row @@ -143,10 +142,8 @@ impl Catalog { let mut sections = Vec::new(); let mut entries = IndexMap::with_capacity(document.len()); let mut alias_lists = Vec::new(); - let mut removals = Vec::new(); for (index, (name, value)) in document.into_iter().enumerate() { if value.is_null() && !is_section(&name) { - removals.push(name.clone()); sections.push(Section { index, key: name, @@ -217,7 +214,6 @@ impl Catalog { store: Store::new(CatalogRows { entries, aliases }), sections, fallback_generalizations, - removals, }, errors: problems, }) @@ -392,11 +388,6 @@ impl Catalog { (catalog, rejections) } - /// Keys written as `null`, which a fetched document uses to retire a row it no longer serves - pub fn removals(&self) -> impl Iterator { - self.removals.iter().map(String::as_str) - } - pub fn unrecognized(&self) -> impl Iterator { self.rows().entries.iter().flat_map(|(name, entry)| { entry diff --git a/litellm-rust/crates/model-catalog/src/layers.rs b/litellm-rust/crates/model-catalog/src/layers.rs index 6b0ae8be17d..2d083c9addf 100644 --- a/litellm-rust/crates/model-catalog/src/layers.rs +++ b/litellm-rust/crates/model-catalog/src/layers.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use indexmap::IndexMap; -use serde_json::Value; use crate::catalog::Catalog; use crate::register::Registrations; @@ -9,13 +8,6 @@ use crate::wire::Fields; const ALIASES: &str = "aliases"; -/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove` -#[derive(Clone, Debug, PartialEq)] -pub enum Patch { - Set(Value), - Remove, -} - /// The four layers served rows are composed from, lowest first #[derive(Clone, Debug)] pub struct Layers { @@ -30,15 +22,6 @@ pub struct Layers { persisted: Registrations, } -impl Patch { - pub fn from_value(value: &Value) -> Self { - match value { - Value::Null => Self::Remove, - value => Self::Set(value.clone()), - } - } -} - impl Layers { pub fn new(bundled: Arc) -> Self { Self { @@ -55,33 +38,14 @@ impl Layers { self.transient.clear(); } - /// What install writes. A fetched row patches the bundled row field by field, a fetched `null` row removes it + /// What install writes. The fetched map replaces bundled whole, bundled only shows through its rejected rows pub fn compose(&self) -> IndexMap { - let mut rows: IndexMap = self - .bundled + self.fetched + .as_deref() + .unwrap_or(&self.bundled) .served() .map(|(key, fields)| (key.to_owned(), without_aliases(fields))) - .collect(); - let Some(fetched) = &self.fetched else { - return rows; - }; - for key in fetched.removals() { - rows.shift_remove(key); - } - for (key, fields) in fetched.served() { - let row = rows.entry(key.to_owned()).or_default(); - for (field, value) in fields.iter().filter(|(field, _)| *field != ALIASES) { - match Patch::from_value(value) { - Patch::Set(value) => { - row.insert(field.clone(), value); - } - Patch::Remove => { - row.shift_remove(field); - } - } - } - } - rows + .collect() } /// The canonical key an alias is served under, from the fetched layer first diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index eba33f4c0ff..31f907689d3 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -374,7 +374,6 @@ fn the_bundled_registry_has_no_problems() { assert!(problems.is_empty(), "{problems:#?}"); const REFERENCE_MODEL_COUNT: usize = 4303; let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value; - assert!(catalog.removals().next().is_none()); assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9); } diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index e17fe352d5d..c809feca5d5 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -1197,22 +1197,24 @@ def test_a_fetched_row_that_fails_validation_is_replaced_by_bundled_or_left_out( assert "gpt-4o" in warnings[0].getMessage() and "acme/new" in warnings[0].getMessage() -def test_a_field_the_fetched_row_lacks_is_served_from_bundled(): +def test_a_field_the_fetched_row_lacks_is_not_served(): module, catalog = _bound_catalog() - bundled_max_output_tokens = module.model_cost["gpt-4o"]["max_output_tokens"] catalog.install(_remote(lambda document: document["gpt-4o"].pop("max_output_tokens"))) - assert module.model_cost["gpt-4o"]["max_output_tokens"] == bundled_max_output_tokens + assert "max_output_tokens" not in module.model_cost["gpt-4o"] -def test_a_key_the_fetched_map_lacks_stays_until_the_map_writes_it_as_null(): +@pytest.mark.parametrize( + "drop", + [lambda document: document.pop("gpt-4o"), lambda document: document.update({"gpt-4o": None})], + ids=["left_out", "null"], +) +def test_a_key_the_fetched_map_does_not_serve_is_removed(drop) -> None: module, catalog = _bound_catalog() - catalog.install(_remote(lambda document: document.pop("gpt-4o"))) - assert "gpt-4o" in module.model_cost + catalog.install(_remote(drop)) - catalog.install(_remote(lambda document: document.update({"gpt-4o": None}))) assert "gpt-4o" not in module.model_cost