From 1216bd6633a26fe4bd40419092f3523e38f8173b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:28:12 -0700 Subject: [PATCH 1/9] fix(cost_map): mark fireworks_ai/minimax-m3 as vision and pin capabilities verified against live calls Fireworks' serverless minimax-m3 answers image inputs, but its model listings say supports_image_input false, so the last two registry syncs flipped supports_vision back to false and the Fireworks pre-flight turned every image request into a 400 without ever calling the provider. Flip both minimax-m3 rows back to supports_vision true and record the live-call evidence in ci_cd/cost_map_pins.json. The cost map guard now fails any PR that lowers a pinned capability, quoting the evidence, and the sync bot may not touch the pins file. --- ci_cd/cost_map_guard.py | 81 +++++++++++-- ci_cd/cost_map_pins.json | 16 +++ ...odel_prices_and_context_window_backup.json | 4 +- model_prices_and_context_window.json | 4 +- tests/unit/test_cost_map_guard.py | 107 +++++++++++++++++- 5 files changed, 192 insertions(+), 20 deletions(-) create mode 100644 ci_cd/cost_map_pins.json diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 5842cf6f1ac..62ce2726c22 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,11 +1,13 @@ """Guard the cost map on pull requests. -Every pull request whose diff against its merge base touches one of the three cost map files gets the file -checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the -map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies -and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot -(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three -files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files or the pins file gets +the file checks: the files parse, the backup copy matches the root file, the JSON schema is in sync and validates the +map, and every capability pinned in ci_cd/cost_map_pins.json (a value verified against a live provider call where the +provider's model listing says otherwise) still holds in the map. A pull request that leaves all of them untouched +skips the checks, since merging it keeps the base branch's copies and its head tree only carries whatever state the +branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the +file checks and additionally may only touch the three cost map files and may only add or update models, so a sync +can never lower a pinned capability or edit the pins. """ from __future__ import annotations @@ -14,7 +16,7 @@ import argparse import json import subprocess import sys -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass from typing import Final @@ -23,10 +25,13 @@ from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" +PINS_PATH: Final = "ci_cd/cost_map_pins.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" CostMap = dict[str, object] +_MISSING: Final = object() @dataclass(frozen=True, slots=True) @@ -34,6 +39,16 @@ class Snapshot: cost_map: str backup: str schema: str + pins: str = "" + + +@dataclass(frozen=True, slots=True) +class Pin: + key: str + field: str + value: object + checked: str + source: str def _parse_object(text: str, path: str) -> CostMap | str: @@ -82,6 +97,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]: return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)} +def _pin(key: str, field: str, spec: object) -> Pin | str: + malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source" + if not isinstance(spec, dict) or "value" not in spec: + return malformed + checked: Final = spec.get("checked") + source: Final = spec.get("source") + if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source: + return malformed + return Pin(key, field, spec["value"], checked, source) + + +def _pins(parsed: CostMap) -> Iterator[Pin | str]: + for key, fields in parsed.items(): + if not isinstance(fields, dict): + yield f"{PINS_PATH}: {key} must map field names to pins" + continue + for field, spec in fields.items(): + yield _pin(key, field, spec) + + +def _pin_violations(parsed: CostMap, entries: dict[str, dict[str, object]]) -> Iterator[str]: + for pin in _pins(parsed): + if isinstance(pin, str): + yield pin + elif pin.key not in entries: + yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry" + elif entries[pin.key].get(pin.field, _MISSING) != pin.value: + yield ( + f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} " + f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first" + ) + + +def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]: + if not pins_text: + return () + parsed: Final = _parse_object(pins_text, PINS_PATH) + if isinstance(parsed, str): + return (parsed,) + return tuple(_pin_violations(parsed, _entries(head_map))) + + def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]: base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) if isinstance(base_map, str): @@ -112,7 +169,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str def touches_cost_map(changed_files: Sequence[str]) -> bool: - return any(path in GUARDED_PATHS for path in changed_files) + return any(path in CHECKED_PATHS for path in changed_files) def contract_for(bot: bool, changed_files: Sequence[str]) -> str: @@ -127,7 +184,11 @@ def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) - return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) + return ( + *_file_failures(head, head_map), + *pin_failures(head.pins, head_map), + *(_bot_failures(base, head_map, changed_files) if bot else ()), + ) def _git(*args: str) -> str | None: @@ -136,7 +197,7 @@ def _git(*args: str) -> str | None: def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) def changed_files(base: str, head: str) -> tuple[str, ...] | None: diff --git a/ci_cd/cost_map_pins.json b/ci_cd/cost_map_pins.json new file mode 100644 index 00000000000..780cd5f1276 --- /dev/null +++ b/ci_cd/cost_map_pins.json @@ -0,0 +1,16 @@ +{ + "fireworks_ai/accounts/fireworks/models/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + }, + "fireworks_ai/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + } +} diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 45f5967d372..6a0f5e08849 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25828,7 +25828,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -26119,7 +26119,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 45f5967d372..6a0f5e08849 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25828,7 +25828,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -26119,7 +26119,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 1595bd9864f..b1b6f2c73e0 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -51,11 +51,16 @@ def _serialize(cost_map: dict[str, object]) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" -def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: str | None = None) -> object: +def _snapshot( + cost_map: dict[str, object], backup: str | None = None, schema: str | None = None, pins: str = "" +) -> object: text = _serialize(cost_map) rendered = schema_module.render(schema_module.build_schema(cost_map)) return guard.Snapshot( - cost_map=text, backup=text if backup is None else backup, schema=rendered if schema is None else schema + cost_map=text, + backup=text if backup is None else backup, + schema=rendered if schema is None else schema, + pins=pins, ) @@ -129,9 +134,9 @@ def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () -@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) -def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: - failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) +@pytest.mark.parametrize("checked_path", guard.CHECKED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(checked_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, checked_path), bot=False) assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] @@ -175,8 +180,76 @@ def test_bot_may_not_change_special_root_keys() -> None: assert _failures(head) == ("bot PRs may not change fallback_generalizations",) -def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: +PINNED_SOURCE: Final = "a 512x512 probe image was read back over the provider's chat endpoint" +PINS: Final = _serialize( + {"openrouter/a": {"supports_vision": {"value": True, "checked": "2026-09-26", "source": PINNED_SOURCE}}} +) + + +def test_a_pinned_capability_that_holds_passes_for_humans_and_bots() -> None: + head = _snapshot(BASE_MAP, pins=PINS) + assert _failures(head, bot=False) == () + assert _failures(head, bot=True) == () + + +@pytest.mark.parametrize("bot", [False, True]) +def test_lowering_a_pinned_capability_fails_with_its_evidence(bot: bool) -> None: + head = _snapshot({**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, pins=PINS) + assert _failures(head, bot=bot) == ( + f"{guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: {PINNED_SOURCE}); " + f"change {guard.PINS_PATH} with new evidence first", + ) + + +def test_dropping_a_pinned_field_or_model_is_reported() -> None: + no_field = _snapshot({**BASE_MAP, "openrouter/a": _entry()}, pins=PINS) + assert [failure for failure in _failures(no_field, bot=False) if "must stay true" in failure] + no_model = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/a"}, pins=PINS) + assert _failures(no_model, bot=False) == ( + f"{guard.PINS_PATH} pins openrouter/a.supports_vision but {guard.COST_MAP_PATH} has no openrouter/a entry", + ) + + +@pytest.mark.parametrize( + ("pins", "expected"), + [ + ( + "{not json", + f"{guard.PINS_PATH} is not valid JSON: Expecting property name enclosed in double quotes: " + "line 1 column 2 (char 1)", + ), + ( + _serialize({"openrouter/a": {"supports_vision": {"value": True}}}), + f"{guard.PINS_PATH}: openrouter/a.supports_vision needs value, checked, and source", + ), + (_serialize({"openrouter/a": True}), f"{guard.PINS_PATH}: openrouter/a must map field names to pins"), + ], +) +def test_malformed_pins_are_reported(pins: str, expected: str) -> None: + assert _failures(_snapshot(BASE_MAP, pins=pins), bot=False) == (expected,) + + +def test_bot_may_not_touch_the_pins_file() -> None: + head = _snapshot(BASE_MAP, pins=PINS) + changed = (*guard.GUARDED_PATHS, guard.PINS_PATH) + assert _failures(head, changed_files=changed, bot=False) == () + assert _failures(head, changed_files=changed) == ( + f"bot PRs may only change the cost map files, not {guard.PINS_PATH}", + ) + + +def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: + cost_map = json.loads((ROOT / guard.COST_MAP_PATH).read_text()) + pins = (ROOT / guard.PINS_PATH).read_text() + assert json.loads(pins), "the pins file must pin at least one capability" + assert guard.pin_failures(pins, cost_map) == () + + +def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str: text = _serialize(cost_map) + if pins is not None: + (repo / guard.PINS_PATH).parent.mkdir(exist_ok=True) + (repo / guard.PINS_PATH).write_text(pins) (repo / guard.COST_MAP_PATH).write_text(text) (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) @@ -271,3 +344,25 @@ def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: head = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head") assert _run_guard(tmp_path, base, head, BOT_REF).returncode == 1 assert _run_guard(tmp_path, base, head, "litellm_fix_pricing").returncode == 0 + + +@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"]) +def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: str) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS) + head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "sync lowers it") + result: Final = _run_guard(tmp_path, base, head, head_ref) + assert result.returncode == 1, result.stdout + result.stderr + pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: " + assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] + + +def test_main_checks_a_pins_only_pr_against_the_head_map(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "base") + (tmp_path / guard.PINS_PATH).parent.mkdir(exist_ok=True) + (tmp_path / guard.PINS_PATH).write_text(PINS) + head: Final = _git_commit(tmp_path, "pin a capability the map disagrees with") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() From e5cc50e7fc0d30ca7f07c1940488c90de48d5810 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:59:30 -0700 Subject: [PATCH 2/9] fix(cost_map): fail the guard when the pins file is missing --- ci_cd/cost_map_guard.py | 5 +++-- tests/unit/test_cost_map_guard.py | 32 ++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 62ce2726c22..538f7a632b5 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -3,7 +3,8 @@ Every pull request whose diff against its merge base touches one of the three cost map files or the pins file gets the file checks: the files parse, the backup copy matches the root file, the JSON schema is in sync and validates the map, and every capability pinned in ci_cd/cost_map_pins.json (a value verified against a live provider call where the -provider's model listing says otherwise) still holds in the map. A pull request that leaves all of them untouched +provider's model listing says otherwise) still holds in the map, so a pull request that deletes the pins file fails +too. A pull request that leaves all of them untouched skips the checks, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch the three cost map files and may only add or update models, so a sync @@ -132,7 +133,7 @@ def _pin_violations(parsed: CostMap, entries: dict[str, dict[str, object]]) -> I def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]: if not pins_text: - return () + return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",) parsed: Final = _parse_object(pins_text, PINS_PATH) if isinstance(parsed, str): return (parsed,) diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index b1b6f2c73e0..67a5f8f69cd 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -51,8 +51,11 @@ def _serialize(cost_map: dict[str, object]) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" +NO_PINS: Final = "{}\n" + + def _snapshot( - cost_map: dict[str, object], backup: str | None = None, schema: str | None = None, pins: str = "" + cost_map: dict[str, object], backup: str | None = None, schema: str | None = None, pins: str = NO_PINS ) -> object: text = _serialize(cost_map) rendered = schema_module.render(schema_module.build_schema(cost_map)) @@ -114,7 +117,7 @@ def test_schema_validation_errors_are_reported() -> None: def test_unclassified_entry_key_is_reported() -> None: text = _serialize({**BASE_MAP, "openrouter/c": _entry(weird_thing=1)}) - head = guard.Snapshot(cost_map=text, backup=text, schema=BASE.schema) + head = guard.Snapshot(cost_map=text, backup=text, schema=BASE.schema, pins=NO_PINS) (failure,) = _failures(head, bot=False) assert "Unclassified keys" in failure and "weird_thing" in failure @@ -229,6 +232,13 @@ def test_malformed_pins_are_reported(pins: str, expected: str) -> None: assert _failures(_snapshot(BASE_MAP, pins=pins), bot=False) == (expected,) +@pytest.mark.parametrize("bot", [False, True]) +def test_a_missing_pins_file_fails(bot: bool) -> None: + assert _failures(_snapshot(BASE_MAP, pins=""), bot=bot) == ( + f"{guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls", + ) + + def test_bot_may_not_touch_the_pins_file() -> None: head = _snapshot(BASE_MAP, pins=PINS) changed = (*guard.GUARDED_PATHS, guard.PINS_PATH) @@ -247,9 +257,10 @@ def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str: text = _serialize(cost_map) - if pins is not None: - (repo / guard.PINS_PATH).parent.mkdir(exist_ok=True) - (repo / guard.PINS_PATH).write_text(pins) + pins_file = repo / guard.PINS_PATH + if pins is not None or not pins_file.exists(): + pins_file.parent.mkdir(exist_ok=True) + pins_file.write_text(NO_PINS if pins is None else pins) (repo / guard.COST_MAP_PATH).write_text(text) (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) @@ -357,6 +368,17 @@ def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: st assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] +def test_main_fails_a_pr_that_deletes_the_pins_file(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS) + subprocess.run(("git", "rm", "-q", guard.PINS_PATH), cwd=tmp_path, check=True) + head: Final = _git_commit(tmp_path, "delete the pins") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + missing: Final = f"- {guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls" + assert missing in result.stdout.splitlines() + + def test_main_checks_a_pins_only_pr_against_the_head_map(tmp_path: Path) -> None: subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) base: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "base") From 126f0db2c3ca5e513b8412eee067f2faf132523d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:10:49 -0700 Subject: [PATCH 3/9] fix(cost_map_guard): skip the pin checks on a branch cut before the pins file existed --- ci_cd/cost_map_guard.py | 5 ++-- tests/unit/test_cost_map_guard.py | 49 ++++++++++++++----------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 292d5068c55..8c94f94566c 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -3,7 +3,8 @@ Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the file checks: the files parse, the backup copy matches the root file, and every capability pinned in ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says -otherwise) still holds in the map, so a pull request that deletes the pins file fails too. A pull request that leaves +otherwise) still holds in the map, so a pull request that deletes the pins file fails too, while a branch cut before +the pins file existed skips the pin checks, since merging it keeps the base branch's pins. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may @@ -205,7 +206,7 @@ def guard_failures( *_file_failures( head, head_map, evaluation_output_price_sources=evaluation_output_price_sources ), - *pin_failures(head.pins, head_map), + *(pin_failures(head.pins, head_map) if head.pins or base.pins else ()), *(_bot_failures(base, head_map, changed_files) if bot else ()), ) diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 6a20a5284ef..b8026c71d10 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -67,9 +67,10 @@ def _failures( changed_files: tuple[str, ...] = MAP_FILES, bot: bool = True, evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES, + base: object = BASE, ) -> tuple[str, ...]: return guard.guard_failures( - BASE, + base, head, changed_files, bot, @@ -256,11 +257,27 @@ def test_malformed_pins_are_reported(pins: str, expected: str) -> None: assert _failures(_snapshot(BASE_MAP, pins=pins), bot=False) == (expected,) +MISSING_PINS: Final = f"{guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls" + + @pytest.mark.parametrize("bot", [False, True]) -def test_a_missing_pins_file_fails(bot: bool) -> None: - assert _failures(_snapshot(BASE_MAP, pins=""), bot=bot) == ( - f"{guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls", - ) +def test_a_pr_that_deletes_the_pins_file_fails(bot: bool) -> None: + head: Final = _snapshot(BASE_MAP, pins="") + assert _failures(head, bot=bot, base=_snapshot(BASE_MAP, pins=PINS)) == (MISSING_PINS,) + + +@pytest.mark.parametrize("bot", [False, True]) +def test_a_branch_cut_before_the_pins_file_existed_skips_the_pin_checks(bot: bool) -> None: + lowered: Final = {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)} + assert _failures(_snapshot(lowered, pins=""), bot=False, base=_snapshot(BASE_MAP, pins="")) == () + assert _failures(_snapshot(BASE_MAP, pins=""), bot=bot, base=_snapshot(BASE_MAP, pins="")) == () + + +def test_a_pr_that_adds_the_pins_file_gets_its_pins_checked() -> None: + head: Final = _snapshot({**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, pins=PINS) + changed: Final = (*MAP_FILES, guard.PINS_PATH) + failures: Final = _failures(head, changed_files=changed, bot=False, base=_snapshot(BASE_MAP, pins="")) + assert [failure for failure in failures if "openrouter/a.supports_vision must stay true" in failure] def test_bot_may_not_touch_the_pins_file() -> None: @@ -388,25 +405,3 @@ def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: st assert result.returncode == 1, result.stdout + result.stderr pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: " assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] - - -def test_main_fails_a_pr_that_deletes_the_pins_file(tmp_path: Path) -> None: - subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) - base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS) - subprocess.run(("git", "rm", "-q", guard.PINS_PATH), cwd=tmp_path, check=True) - head: Final = _git_commit(tmp_path, "delete the pins") - result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") - assert result.returncode == 1, result.stdout + result.stderr - missing: Final = f"- {guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls" - assert missing in result.stdout.splitlines() - - -def test_main_checks_a_pins_only_pr_against_the_head_map(tmp_path: Path) -> None: - subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) - base: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "base") - (tmp_path / guard.PINS_PATH).parent.mkdir(exist_ok=True) - (tmp_path / guard.PINS_PATH).write_text(PINS) - head: Final = _git_commit(tmp_path, "pin a capability the map disagrees with") - result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") - assert result.returncode == 1, result.stdout + result.stderr - assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() From 2be59c8e4c865073d8892f18404071079487453b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:46:13 -0700 Subject: [PATCH 4/9] fix(ci): check cost map pins against the merge result and keep price sync PRs off the pins file --- .github/CODEOWNERS | 1 + .github/workflows/cost-map-guard.yml | 7 ++- ci_cd/cost_map_guard.py | 42 ++++++++++++-- tests/unit/test_cost_map_guard.py | 84 ++++++++++++++++++++++++---- 4 files changed, 116 insertions(+), 18 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6c06f7d83f..1fa139a13e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,5 @@ /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index f9172b9551a..143436372a4 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - - name: Fetch the pull request head and its merge base + - name: Fetch the pull request head, its merge base, and the base branch tip id: revisions env: GH_TOKEN: ${{ github.token }} @@ -28,7 +28,7 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" - git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" + git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$BASE_SHA" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries @@ -37,7 +37,8 @@ jobs: - name: Run the guard env: MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF" + uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --target "$BASE_SHA" --head-ref "$HEAD_REF" diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 8c94f94566c..d8aa43aed15 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -3,12 +3,13 @@ Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the file checks: the files parse, the backup copy matches the root file, and every capability pinned in ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says -otherwise) still holds in the map, so a pull request that deletes the pins file fails too, while a branch cut before -the pins file existed skips the pin checks, since merging it keeps the base branch's pins. A pull request that leaves +otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before +a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may -only add or update models, so a sync can never lower a pinned capability or edit the pins. +only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests +(branches named litellm-providers/price-sync*) get the human checks and may not touch the pins file either. """ from __future__ import annotations @@ -28,6 +29,7 @@ PINS_PATH: Final = "ci_cd/cost_map_pins.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" +PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) @@ -179,6 +181,12 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def price_sync_failures(head_ref: str, changed_files: Sequence[str]) -> tuple[str, ...]: + if not head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX) or PINS_PATH not in changed_files: + return () + return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + + def touches_cost_map(changed_files: Sequence[str]) -> bool: return any(path in CHECKED_PATHS for path in changed_files) @@ -189,9 +197,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str: return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" +def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]: + if merged is None: + return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",) + if not merged.pins and not base.pins: + return () + merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH) + if isinstance(merged_map, str): + return (f"after merging into the base branch, {merged_map}",) + return pin_failures(merged.pins, merged_map) + + def guard_failures( base: Snapshot, head: Snapshot, + merged: Snapshot | None, changed_files: Sequence[str], bot: bool, *, @@ -206,7 +226,7 @@ def guard_failures( *_file_failures( head, head_map, evaluation_output_price_sources=evaluation_output_price_sources ), - *(pin_failures(head.pins, head_map) if head.pins or base.pins else ()), + *_merged_pin_failures(base, merged), *(_bot_failures(base, head_map, changed_files) if bot else ()), ) @@ -220,6 +240,11 @@ def snapshot(revision: str) -> Snapshot: return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) +def merged_tree(merge_base: str, target: str, head: str) -> str | None: + output: Final = _git("merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head) + return None if output is None else output.split("\n", 1)[0] + + def changed_files(base: str, head: str) -> tuple[str, ...] | None: diff: Final = _git("diff", "--name-only", "--no-renames", base, head) return None if diff is None else tuple(diff.splitlines()) @@ -229,6 +254,7 @@ def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", required=True, help="merge base of the pull request") parser.add_argument("--head", required=True, help="head commit of the pull request") + parser.add_argument("--target", required=True, help="tip of the base branch the pull request merges into") parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) @@ -236,7 +262,13 @@ def main(argv: Sequence[str]) -> int: if changed is None: print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") return 1 - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + merged: Final = merged_tree(args.base, args.target, args.head) + failures: Final = ( + *guard_failures( + snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot + ), + *price_sync_failures(args.head_ref, changed), + ) contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index b8026c71d10..99e5bc8e743 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -60,6 +60,7 @@ def _snapshot(cost_map: dict[str, object], backup: str | None = None, pins: str BASE: Final = _snapshot(BASE_MAP) +UP_TO_DATE: Final = object() def _failures( @@ -68,10 +69,12 @@ def _failures( bot: bool = True, evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES, base: object = BASE, + merged: object = UP_TO_DATE, ) -> tuple[str, ...]: return guard.guard_failures( base, head, + head if merged is UP_TO_DATE else merged, changed_files, bot, evaluation_output_price_sources=evaluation_output_price_sources, @@ -220,13 +223,16 @@ def test_a_pinned_capability_that_holds_passes_for_humans_and_bots() -> None: assert _failures(head, bot=True) == () +LOWERED: Final = ( + f"{guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: {PINNED_SOURCE}); " + f"change {guard.PINS_PATH} with new evidence first" +) +NO_VISION_MAP: Final = {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)} + + @pytest.mark.parametrize("bot", [False, True]) def test_lowering_a_pinned_capability_fails_with_its_evidence(bot: bool) -> None: - head = _snapshot({**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, pins=PINS) - assert _failures(head, bot=bot) == ( - f"{guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: {PINNED_SOURCE}); " - f"change {guard.PINS_PATH} with new evidence first", - ) + assert _failures(_snapshot(NO_VISION_MAP, pins=PINS), bot=bot) == (LOWERED,) def test_dropping_a_pinned_field_or_model_is_reported() -> None: @@ -267,10 +273,37 @@ def test_a_pr_that_deletes_the_pins_file_fails(bot: bool) -> None: @pytest.mark.parametrize("bot", [False, True]) -def test_a_branch_cut_before_the_pins_file_existed_skips_the_pin_checks(bot: bool) -> None: - lowered: Final = {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)} - assert _failures(_snapshot(lowered, pins=""), bot=False, base=_snapshot(BASE_MAP, pins="")) == () - assert _failures(_snapshot(BASE_MAP, pins=""), bot=bot, base=_snapshot(BASE_MAP, pins="")) == () +def test_pin_checks_are_skipped_when_no_side_of_the_merge_has_the_pins_file(bot: bool) -> None: + assert _failures(_snapshot(NO_VISION_MAP, pins=""), bot=bot, base=_snapshot(NO_VISION_MAP, pins="")) == () + + +@pytest.mark.parametrize("bot", [False, True]) +def test_a_branch_cut_before_a_pin_was_added_cannot_lower_it(bot: bool) -> None: + head: Final = _snapshot(NO_VISION_MAP, pins="") + merged: Final = _snapshot(NO_VISION_MAP, pins=PINS) + assert _failures(head, bot=bot, base=_snapshot(BASE_MAP, pins=""), merged=merged) == (LOWERED,) + + +@pytest.mark.parametrize("bot", [False, True]) +def test_a_branch_cut_before_a_pin_was_added_passes_when_merging_keeps_the_pinned_value(bot: bool) -> None: + repriced: Final = {"openrouter/b": _entry(3e-06)} + head: Final = _snapshot({**NO_VISION_MAP, **repriced}, pins="") + merged: Final = _snapshot({**BASE_MAP, **repriced}, pins=PINS) + assert _failures(head, bot=bot, base=_snapshot(NO_VISION_MAP, pins=""), merged=merged) == () + + +def test_a_pr_that_does_not_merge_cleanly_is_told_to_merge_the_base_branch_in() -> None: + assert _failures(BASE, bot=False, merged=None) == ( + "the pull request does not merge cleanly into its base branch; merge the base branch in first", + ) + + +def test_a_merge_result_that_is_not_valid_json_is_reported() -> None: + merged: Final = guard.Snapshot(cost_map="{broken", backup="{broken", pins=PINS) + assert _failures(BASE, bot=False, merged=merged) == ( + f"after merging into the base branch, {guard.COST_MAP_PATH} is not valid JSON: " + "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)", + ) def test_a_pr_that_adds_the_pins_file_gets_its_pins_checked() -> None: @@ -289,6 +322,26 @@ def test_bot_may_not_touch_the_pins_file() -> None: ) +@pytest.mark.parametrize( + ("head_ref", "changed", "expected"), + [ + ( + "litellm-providers/price-sync-fireworks-ai", + (*MAP_FILES, guard.PINS_PATH), + ( + f"price sync PRs may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence", + ), + ), + ("litellm-providers/price-sync-fireworks-ai", MAP_FILES, ()), + ("fix-openrouter-price-sync", (*MAP_FILES, guard.PINS_PATH), ()), + ], +) +def test_price_sync_prs_may_not_touch_the_pins_file( + head_ref: str, changed: tuple[str, ...], expected: tuple[str, ...] +) -> None: + assert guard.price_sync_failures(head_ref, changed) == expected + + def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: cost_map = json.loads((ROOT / guard.COST_MAP_PATH).read_text()) pins = (ROOT / guard.PINS_PATH).read_text() @@ -328,7 +381,18 @@ def _git_commit(repo: Path, message: str) -> str: def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.CompletedProcess[str]: return subprocess.run( - (sys.executable, str(CI_CD / "cost_map_guard.py"), "--base", base, "--head", head, "--head-ref", head_ref), + ( + sys.executable, + str(CI_CD / "cost_map_guard.py"), + "--base", + base, + "--head", + head, + "--target", + base, + "--head-ref", + head_ref, + ), cwd=repo, capture_output=True, text=True, From 00e4420fd006872a7d20ceec6fc2b9a5ab15a8e9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:40:07 -0700 Subject: [PATCH 5/9] fix(ci): run the cost map guard against the live base branch tip and re-run it when main's pins change --- .github/workflows/cost-map-guard.yml | 12 +++--- .github/workflows/cost-map-pins-recheck.yml | 43 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cost-map-pins-recheck.yml diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 143436372a4..fab84afe657 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -24,12 +24,14 @@ jobs: id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" - git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$BASE_SHA" + target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')" + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries with: @@ -37,8 +39,8 @@ jobs: - name: Run the guard env: MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + TARGET: ${{ steps.revisions.outputs.target }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --target "$BASE_SHA" --head-ref "$HEAD_REF" + uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --target "$TARGET" --head-ref "$HEAD_REF" diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml new file mode 100644 index 00000000000..b1ebbb5c1a2 --- /dev/null +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -0,0 +1,43 @@ +name: Cost map pins recheck + +on: + push: + branches: + - main + paths: + - ci_cd/cost_map_pins.json + +permissions: {} + +jobs: + rerun-cost-map-guard: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: Re-run the cost map guard on open pull requests that touch the cost map or its pins + env: + GH_TOKEN: ${{ github.token }} + BASE_REF: ${{ github.ref_name }} + run: | + gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \ + -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) { + nodes { headRefOid files(first: 100) { nodes { path } } } + pageInfo { hasNextPage endCursor } + } + } + }' \ + --jq '.data.repository.pullRequests.nodes[] | select(any(.files.nodes[].path; IN("model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"))) | .headRefOid' | + while read -r head_sha; do + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + if [ -z "$run_id" ]; then + echo "::warning::no cost map guard run found for $head_sha" + continue + fi + gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY" || echo "::warning::could not re-run cost map guard run $run_id for $head_sha" + done From e4e03547e0fd395ffba0b00cf8c1ef1f4e8c257c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:57:56 -0700 Subject: [PATCH 6/9] fix(ci): pass the live base tip to the cost map guard through env and re-run every guard run on a pin change main now runs the guard on pull_request with the base branch's copy of the script, so the workflow passes the live tip in COST_MAP_GUARD_TARGET, which the older script ignores. The recheck reads past GraphQL's 100-file cap, re-runs every guard run for a shared head, and fails when a re-run cannot start. The guard tells a merge conflict apart from a failed merge-tree call. --- .github/workflows/cost-map-guard.yml | 4 +- .github/workflows/cost-map-pins-recheck.yml | 46 +++++++++++++++++---- ci_cd/cost_map_guard.py | 35 ++++++++++++++-- tests/unit/test_cost_map_guard.py | 45 ++++++++++++++++++-- 4 files changed, 112 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 53972618e13..3e8bb572530 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -40,8 +40,8 @@ jobs: - name: Run the guard env: MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} - TARGET: ${{ steps.revisions.outputs.target }} + COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --target "$TARGET" --head-ref "$HEAD_REF" + uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF" diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml index b1ebbb5c1a2..88ce773e4f3 100644 --- a/.github/workflows/cost-map-pins-recheck.yml +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -22,22 +22,50 @@ jobs: env: GH_TOKEN: ${{ github.token }} BASE_REF: ${{ github.ref_name }} + COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]' run: | - gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \ + candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \ -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) { repository(owner: $owner, name: $name) { pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) { - nodes { headRefOid files(first: 100) { nodes { path } } } + nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } } pageInfo { hasNextPage endCursor } } } }' \ - --jq '.data.repository.pullRequests.nodes[] | select(any(.files.nodes[].path; IN("model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"))) | .headRefOid' | - while read -r head_sha; do - run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty')" - if [ -z "$run_id" ]; then - echo "::warning::no cost map guard run found for $head_sha" + --jq '(env.COST_MAP_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[] + | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)} + | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')" + failed=0 + declare -A rerun_heads + while read -r number head_sha listed; do + [ -n "$number" ] || continue + if [ "$listed" = false ]; then + if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \ + --jq '(env.COST_MAP_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then + echo "::error::could not list the files of #$number" + failed=1 + continue + fi + [ -n "$changed" ] || continue + fi + [ -z "${rerun_heads[$head_sha]:-}" ] || continue + rerun_heads[$head_sha]=1 + if ! run_ids="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list the cost map guard runs of #$number" + failed=1 continue fi - gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY" || echo "::warning::could not re-run cost map guard run $run_id for $head_sha" - done + if [ -z "$run_ids" ]; then + echo "::warning::no cost map guard run found for #$number" + continue + fi + for run_id in $run_ids; do + gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true + if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then + echo "::error::could not re-run cost map guard run $run_id for #$number" + failed=1 + fi + done + done <<< "$candidates" + exit "$failed" diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index d8aa43aed15..d48ab9b85e3 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -16,6 +16,7 @@ from __future__ import annotations import argparse import json +import os import subprocess import sys from collections.abc import Iterator, Mapping, Sequence @@ -30,6 +31,8 @@ GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" +TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" +MERGE_CONFLICT_EXIT_CODE: Final = 1 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) @@ -44,6 +47,11 @@ class Snapshot: pins: str +@dataclass(frozen=True, slots=True) +class MergeFailed: + stderr: str + + @dataclass(frozen=True, slots=True) class Pin: key: str @@ -240,9 +248,19 @@ def snapshot(revision: str) -> Snapshot: return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) -def merged_tree(merge_base: str, target: str, head: str) -> str | None: - output: Final = _git("merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head) - return None if output is None else output.split("\n", 1)[0] +def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None: + result: Final = subprocess.run( + ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head), + check=False, + capture_output=True, + text=True, + ) + tree: Final = result.stdout.split("\n", 1)[0] + if result.returncode == 0: + return tree + if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree: + return None + return MergeFailed(result.stderr.strip()) def changed_files(base: str, head: str) -> tuple[str, ...] | None: @@ -254,7 +272,13 @@ def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", required=True, help="merge base of the pull request") parser.add_argument("--head", required=True, help="head commit of the pull request") - parser.add_argument("--target", required=True, help="tip of the base branch the pull request merges into") + env_target: Final = os.environ.get(TARGET_ENV) + parser.add_argument( + "--target", + default=env_target, + required=env_target is None, + help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}", + ) parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) @@ -263,6 +287,9 @@ def main(argv: Sequence[str]) -> int: print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") return 1 merged: Final = merged_tree(args.base, args.target, args.head) + if isinstance(merged, MergeFailed): + print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}") + return 1 failures: Final = ( *guard_failures( snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 99e5bc8e743..e9642fae87c 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -1,5 +1,6 @@ import importlib.util import json +import os import subprocess import sys from collections.abc import Mapping @@ -379,7 +380,10 @@ def _git_commit(repo: Path, message: str) -> str: ).stdout.strip() -def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.CompletedProcess[str]: +def _run_guard( + repo: Path, base: str, head: str, head_ref: str, *, target: str | None = None, target_env: str | None = None +) -> subprocess.CompletedProcess[str]: + target_args: Final = () if target_env is not None else ("--target", target or base) return subprocess.run( ( sys.executable, @@ -388,8 +392,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co base, "--head", head, - "--target", - base, + *target_args, "--head-ref", head_ref, ), @@ -397,6 +400,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co capture_output=True, text=True, check=False, + env=None if target_env is None else {**os.environ, guard.TARGET_ENV: target_env}, ) @@ -469,3 +473,38 @@ def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: st assert result.returncode == 1, result.stdout + result.stderr pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: " assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] + + +@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"]) +def test_main_holds_a_pin_the_base_branch_added_after_the_pr_branched(tmp_path: Path, head_ref: str) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base") + target: Final = _commit(tmp_path, BASE_MAP, "pin on the base branch", pins=PINS) + subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True) + head: Final = _commit(tmp_path, NO_VISION_MAP, "branch cut before the pin lowers it", pins=NO_PINS) + result: Final = _run_guard(tmp_path, merge_base, head, head_ref, target_env=target) + assert result.returncode == 1, result.stdout + result.stderr + pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: " + assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] + + +def test_main_reports_a_failed_merge_apart_from_a_conflict(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing", target="0" * 40) + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.startswith(f"cost map guard failed: git merge-tree into {'0' * 40} failed: ") + + +def test_main_tells_a_conflicting_pr_to_merge_the_base_branch_in(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base") + target: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(3e-06)}, "base branch reprices b") + subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True) + head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(4e-06)}, "pr reprices b differently") + result: Final = _run_guard(tmp_path, merge_base, head, "litellm_fix_pricing", target=target) + assert result.returncode == 1, result.stdout + result.stderr + conflict_line: Final = "- the pull request does not merge cleanly into its base branch; merge the base branch in first" + assert conflict_line in result.stdout.splitlines() + From f20a86fd399b13dc8389eb39091206a73477d97f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:22:20 -0700 Subject: [PATCH 7/9] fix(ci): run the cost map pins recheck on litellm_ base branches too --- .github/workflows/cost-map-pins-recheck.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml index 88ce773e4f3..d5d84237c01 100644 --- a/.github/workflows/cost-map-pins-recheck.yml +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "litellm_**" paths: - ci_cd/cost_map_pins.json From 6d68c41fc72629cf00dcdd928f825a5c197e5550 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:01:28 -0700 Subject: [PATCH 8/9] fix(ci): bar bot-opened PRs from the pins, recheck pin PRs on cost map changes, and skip the merge for untouched PRs --- .github/workflows/cost-map-guard.yml | 1 + .github/workflows/cost-map-pins-recheck.yml | 29 ++++++--- ci_cd/cost_map_guard.py | 20 +++++-- tests/unit/test_cost_map_guard.py | 66 ++++++++++++++++----- 4 files changed, 89 insertions(+), 27 deletions(-) diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 3e8bb572530..6519b65e71b 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -41,6 +41,7 @@ jobs: env: MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }} + COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml index d5d84237c01..02cd6633ae2 100644 --- a/.github/workflows/cost-map-pins-recheck.yml +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -6,6 +6,8 @@ on: - main - "litellm_**" paths: + - model_prices_and_context_window.json + - litellm/model_prices_and_context_window_backup.json - ci_cd/cost_map_pins.json permissions: {} @@ -19,12 +21,21 @@ jobs: contents: read pull-requests: read steps: - - name: Re-run the cost map guard on open pull requests that touch the cost map or its pins + - name: Re-run the cost map guard on open pull requests the push could break env: GH_TOKEN: ${{ github.token }} BASE_REF: ${{ github.ref_name }} COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]' + PINS_PATHS: '["ci_cd/cost_map_pins.json"]' run: | + if jq -e --arg pins ci_cd/cost_map_pins.json \ + '(.commits | length) as $n | $n == 0 or $n >= 2048 or any(.commits[]; ((.added // []) + (.modified // []) + (.removed // [])) | index($pins))' \ + "$GITHUB_EVENT_PATH" > /dev/null; then + export RECHECK_PATHS="$COST_MAP_PATHS" + else + export RECHECK_PATHS="$PINS_PATHS" + fi + rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)" candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \ -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) { repository(owner: $owner, name: $name) { @@ -34,7 +45,7 @@ jobs: } } }' \ - --jq '(env.COST_MAP_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[] + --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[] | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)} | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')" failed=0 @@ -43,7 +54,7 @@ jobs: [ -n "$number" ] || continue if [ "$listed" = false ]; then if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \ - --jq '(env.COST_MAP_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then + --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then echo "::error::could not list the files of #$number" failed=1 continue @@ -52,21 +63,25 @@ jobs: fi [ -z "${rerun_heads[$head_sha]:-}" ] || continue rerun_heads[$head_sha]=1 - if ! run_ids="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId --jq '.[].databaseId')"; then + if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then echo "::error::could not list the cost map guard runs of #$number" failed=1 continue fi - if [ -z "$run_ids" ]; then + if [ -z "$runs" ]; then echo "::warning::no cost map guard run found for #$number" continue fi - for run_id in $run_ids; do + while read -r run_id created_at; do + if [[ "$created_at" < "$rerun_cutoff" ]]; then + echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push" + continue + fi gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then echo "::error::could not re-run cost map guard run $run_id for #$number" failed=1 fi - done + done <<< "$runs" done <<< "$candidates" exit "$failed" diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index d48ab9b85e3..d5fd755ba18 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -9,7 +9,8 @@ all three untouched skips them, since merging it keeps the base branch's copies whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests -(branches named litellm-providers/price-sync*) get the human checks and may not touch the pins file either. +(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is +Bot) get the human checks and may not touch the pins file either. """ from __future__ import annotations @@ -32,6 +33,8 @@ CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" +AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE" +BOT_AUTHOR_TYPE: Final = "Bot" MERGE_CONFLICT_EXIT_CODE: Final = 1 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) @@ -189,10 +192,14 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) -def price_sync_failures(head_ref: str, changed_files: Sequence[str]) -> tuple[str, ...]: - if not head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX) or PINS_PATH not in changed_files: +def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]: + if PINS_PATH not in changed_files: return () - return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX): + return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + if author_type == BOT_AUTHOR_TYPE: + return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + return () def touches_cost_map(changed_files: Sequence[str]) -> bool: @@ -286,6 +293,9 @@ def main(argv: Sequence[str]) -> int: if changed is None: print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") return 1 + if not bot and not touches_cost_map(changed): + print(f"cost map guard passed ({contract_for(bot, changed)})") + return 0 merged: Final = merged_tree(args.base, args.target, args.head) if isinstance(merged, MergeFailed): print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}") @@ -294,7 +304,7 @@ def main(argv: Sequence[str]) -> int: *guard_failures( snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot ), - *price_sync_failures(args.head_ref, changed), + *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed), ) contract: Final = contract_for(bot, changed) if failures: diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index e9642fae87c..2602149b623 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -323,24 +323,29 @@ def test_bot_may_not_touch_the_pins_file() -> None: ) +PRICE_SYNC_PIN_FAILURE: Final = ( + f"price sync PRs may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence" +) +BOT_AUTHOR_PIN_FAILURE: Final = ( + f"PRs opened by a bot may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence" +) + + @pytest.mark.parametrize( - ("head_ref", "changed", "expected"), + ("head_ref", "author_type", "changed", "expected"), [ - ( - "litellm-providers/price-sync-fireworks-ai", - (*MAP_FILES, guard.PINS_PATH), - ( - f"price sync PRs may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence", - ), - ), - ("litellm-providers/price-sync-fireworks-ai", MAP_FILES, ()), - ("fix-openrouter-price-sync", (*MAP_FILES, guard.PINS_PATH), ()), + ("litellm-providers/price-sync-fireworks-ai", "User", (*MAP_FILES, guard.PINS_PATH), (PRICE_SYNC_PIN_FAILURE,)), + ("litellm-providers/price-sync-fireworks-ai", "Bot", MAP_FILES, ()), + ("litellm_azure_ai_mai_image_2_5_pro", "Bot", (*MAP_FILES, guard.PINS_PATH), (BOT_AUTHOR_PIN_FAILURE,)), + ("litellm_azure_ai_mai_image_2_5_pro", "Bot", MAP_FILES, ()), + ("fix-openrouter-price-sync", "User", (*MAP_FILES, guard.PINS_PATH), ()), + ("fix-openrouter-price-sync", "", (*MAP_FILES, guard.PINS_PATH), ()), ], ) -def test_price_sync_prs_may_not_touch_the_pins_file( - head_ref: str, changed: tuple[str, ...], expected: tuple[str, ...] +def test_only_human_opened_non_price_sync_prs_may_touch_the_pins_file( + head_ref: str, author_type: str, changed: tuple[str, ...], expected: tuple[str, ...] ) -> None: - assert guard.price_sync_failures(head_ref, changed) == expected + assert guard.pin_author_failures(head_ref, author_type, changed) == expected def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: @@ -381,9 +386,20 @@ def _git_commit(repo: Path, message: str) -> str: def _run_guard( - repo: Path, base: str, head: str, head_ref: str, *, target: str | None = None, target_env: str | None = None + repo: Path, + base: str, + head: str, + head_ref: str, + *, + target: str | None = None, + target_env: str | None = None, + author_type: str | None = None, ) -> subprocess.CompletedProcess[str]: target_args: Final = () if target_env is not None else ("--target", target or base) + env_overrides: Final = { + **({} if target_env is None else {guard.TARGET_ENV: target_env}), + **({} if author_type is None else {guard.AUTHOR_TYPE_ENV: author_type}), + } return subprocess.run( ( sys.executable, @@ -400,7 +416,7 @@ def _run_guard( capture_output=True, text=True, check=False, - env=None if target_env is None else {**os.environ, guard.TARGET_ENV: target_env}, + env={**os.environ, **env_overrides}, ) @@ -508,3 +524,23 @@ def test_main_tells_a_conflicting_pr_to_merge_the_base_branch_in(tmp_path: Path) conflict_line: Final = "- the pull request does not merge cleanly into its base branch; merge the base branch in first" assert conflict_line in result.stdout.splitlines() + +def test_main_skips_the_merge_for_a_pr_that_leaves_the_cost_map_alone(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + head: Final = _commit_code_only(tmp_path, "code change") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing", target="0" * 40) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.splitlines() == ["cost map guard passed (human PR, cost map untouched)"] + + +@pytest.mark.parametrize(("author_type", "expected_code"), [("Bot", 1), ("User", 0)]) +def test_main_lets_only_a_human_opened_pr_change_the_pins(tmp_path: Path, author_type: str, expected_code: int) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS) + head: Final = _commit(tmp_path, NO_VISION_MAP, "lower the flag and drop its pin", pins=NO_PINS) + result: Final = _run_guard( + tmp_path, base, head, "litellm_azure_ai_mai_image_2_5_pro", author_type=author_type + ) + assert result.returncode == expected_code, result.stdout + result.stderr + assert (f"- {BOT_AUTHOR_PIN_FAILURE}" in result.stdout.splitlines()) == (author_type == "Bot") From b666f9d8b93076b9137155e58566c5b1f732ef9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:34:05 -0700 Subject: [PATCH 9/9] fix(ci): read the pushed files from the compare API in the cost map pins recheck The push event GitHub hands to Actions leaves each commit's added, modified and removed lists null, so the recheck never saw a pins change and re-ran only the PRs that touch the pins. Compare the push's before and after commits instead, and re-check every cost map PR when the branch is new, the push was forced, the compare fails, or it lists 300 files, the most it returns. --- .github/workflows/cost-map-pins-recheck.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml index 02cd6633ae2..f2572efbb31 100644 --- a/.github/workflows/cost-map-pins-recheck.yml +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -25,12 +25,16 @@ jobs: env: GH_TOKEN: ${{ github.token }} BASE_REF: ${{ github.ref_name }} + BEFORE_SHA: ${{ github.event.before }} + AFTER_SHA: ${{ github.event.after }} + FORCED: ${{ github.event.forced }} COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]' PINS_PATHS: '["ci_cd/cost_map_pins.json"]' run: | - if jq -e --arg pins ci_cd/cost_map_pins.json \ - '(.commits | length) as $n | $n == 0 or $n >= 2048 or any(.commits[]; ((.added // []) + (.modified // []) + (.removed // [])) | index($pins))' \ - "$GITHUB_EVENT_PATH" > /dev/null; then + if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \ + || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \ + --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \ + | grep -qx true; then export RECHECK_PATHS="$COST_MAP_PATHS" else export RECHECK_PATHS="$PINS_PATHS"