From 0ea1f99c0983f06512fd8d1a6da4ff6988d8fbfe Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Thu, 24 Sep 2026 19:27:55 +0000 Subject: [PATCH] fix(ci): skip cost-map schema checks when the PR changes the generator cost-map-guard runs the base branch's generator, so a diff that edits ci_cd/generate_model_prices_schema.py makes its schema sync and validation checks stale by definition. The parse and backup checks still run, and the head generator is covered by the schema unit tests. --- ci_cd/cost_map_guard.py | 21 +++++++++++++++------ tests/test_litellm/test_cost_map_guard.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 5842cf6f1ac..346c3cf0cb8 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -5,7 +5,9 @@ checks: the files parse, the backup copy matches the root file, and the JSON sch map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three -files and may only add or update models. +files and may only add or update models. When the diff also changes the schema generator the sync and +validation checks are skipped, since this check runs the base branch's generator and the unit tests cover the +head's. """ from __future__ import annotations @@ -23,6 +25,7 @@ from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" +GENERATOR_PATH: Final = "ci_cd/generate_model_prices_schema.py" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" @@ -51,15 +54,17 @@ def _rendered_schema(cost_map: CostMap) -> str: return str(error) -def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]: - schema_text: Final = _rendered_schema(head_map) - if not schema_text.startswith("{"): - return (schema_text,) +def _file_failures(head: Snapshot, head_map: CostMap, generator_changed: bool) -> tuple[str, ...]: backup_failure: Final = ( () if head.backup == head.cost_map else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",) ) + if generator_changed: + return backup_failure + schema_text: Final = _rendered_schema(head_map) + if not schema_text.startswith("{"): + return (schema_text,) schema_failure: Final = ( () if head.schema == schema_text @@ -127,7 +132,11 @@ def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) - return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) + generator_changed: Final = GENERATOR_PATH in changed_files + return ( + *_file_failures(head, head_map, generator_changed), + *(_bot_failures(base, head_map, changed_files) if bot else ()), + ) def _git(*args: str) -> str | None: diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1595bd9864f..03b35d87606 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -98,6 +98,20 @@ def test_schema_out_of_sync_is_reported() -> None: assert [failure for failure in _failures(head, bot=False) if failure.startswith(guard.SCHEMA_PATH)] +def test_changing_the_generator_skips_the_schema_checks_but_keeps_the_backup_check() -> None: + """A PR that edits the generator makes the base branch's copy stale, so the schema sync + and validation checks do not run; the backup drift check still does.""" + out_of_sync: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry(supports_audio_input=True)}, schema=BASE.schema) + with_generator: Final = (*MAP_FILES, guard.GENERATOR_PATH) + assert _failures(out_of_sync, changed_files=with_generator, bot=False) == () + assert [failure for failure in _failures(out_of_sync, bot=False) if failure.startswith(guard.SCHEMA_PATH)] + drifted: Final = guard.Snapshot( + cost_map=out_of_sync.cost_map, backup="{drifted", schema=out_of_sync.schema + ) + (failure,) = _failures(drifted, changed_files=with_generator, bot=False) + assert failure.startswith(guard.BACKUP_PATH) + + def test_schema_validation_errors_are_reported() -> None: head = _snapshot({**BASE_MAP, "openrouter/c": _entry(-1e-06)}) prefix = f"{guard.COST_MAP_PATH} does not validate against its schema: openrouter/c."