diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 5842cf6f1ac..62ce2726c22 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,11 +1,13 @@ """Guard the cost map on pull requests. -Every pull request whose diff against its merge base touches one of the three cost map files gets the file -checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the -map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies -and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot -(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three -files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files or the pins file gets +the file checks: the files parse, the backup copy matches the root file, the JSON schema is in sync and validates the +map, and every capability pinned in ci_cd/cost_map_pins.json (a value verified against a live provider call where the +provider's model listing says otherwise) still holds in the map. A pull request that leaves all of them untouched +skips the checks, since merging it keeps the base branch's copies and its head tree only carries whatever state the +branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always get the +file checks and additionally may only touch the three cost map files and may only add or update models, so a sync +can never lower a pinned capability or edit the pins. """ from __future__ import annotations @@ -14,7 +16,7 @@ import argparse import json import subprocess import sys -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass from typing import Final @@ -23,10 +25,13 @@ from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" +PINS_PATH: Final = "ci_cd/cost_map_pins.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" CostMap = dict[str, object] +_MISSING: Final = object() @dataclass(frozen=True, slots=True) @@ -34,6 +39,16 @@ class Snapshot: cost_map: str backup: str schema: str + pins: str = "" + + +@dataclass(frozen=True, slots=True) +class Pin: + key: str + field: str + value: object + checked: str + source: str def _parse_object(text: str, path: str) -> CostMap | str: @@ -82,6 +97,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]: return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)} +def _pin(key: str, field: str, spec: object) -> Pin | str: + malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source" + if not isinstance(spec, dict) or "value" not in spec: + return malformed + checked: Final = spec.get("checked") + source: Final = spec.get("source") + if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source: + return malformed + return Pin(key, field, spec["value"], checked, source) + + +def _pins(parsed: CostMap) -> Iterator[Pin | str]: + for key, fields in parsed.items(): + if not isinstance(fields, dict): + yield f"{PINS_PATH}: {key} must map field names to pins" + continue + for field, spec in fields.items(): + yield _pin(key, field, spec) + + +def _pin_violations(parsed: CostMap, entries: dict[str, dict[str, object]]) -> Iterator[str]: + for pin in _pins(parsed): + if isinstance(pin, str): + yield pin + elif pin.key not in entries: + yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry" + elif entries[pin.key].get(pin.field, _MISSING) != pin.value: + yield ( + f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} " + f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first" + ) + + +def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]: + if not pins_text: + return () + parsed: Final = _parse_object(pins_text, PINS_PATH) + if isinstance(parsed, str): + return (parsed,) + return tuple(_pin_violations(parsed, _entries(head_map))) + + def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]: base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) if isinstance(base_map, str): @@ -112,7 +169,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str def touches_cost_map(changed_files: Sequence[str]) -> bool: - return any(path in GUARDED_PATHS for path in changed_files) + return any(path in CHECKED_PATHS for path in changed_files) def contract_for(bot: bool, changed_files: Sequence[str]) -> str: @@ -127,7 +184,11 @@ def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) - return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) + return ( + *_file_failures(head, head_map), + *pin_failures(head.pins, head_map), + *(_bot_failures(base, head_map, changed_files) if bot else ()), + ) def _git(*args: str) -> str | None: @@ -136,7 +197,7 @@ def _git(*args: str) -> str | None: def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) def changed_files(base: str, head: str) -> tuple[str, ...] | None: diff --git a/ci_cd/cost_map_pins.json b/ci_cd/cost_map_pins.json new file mode 100644 index 00000000000..780cd5f1276 --- /dev/null +++ b/ci_cd/cost_map_pins.json @@ -0,0 +1,16 @@ +{ + "fireworks_ai/accounts/fireworks/models/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + }, + "fireworks_ai/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + } +} diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 45f5967d372..6a0f5e08849 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25828,7 +25828,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -26119,7 +26119,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 45f5967d372..6a0f5e08849 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25828,7 +25828,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -26119,7 +26119,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 1595bd9864f..b1b6f2c73e0 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -51,11 +51,16 @@ def _serialize(cost_map: dict[str, object]) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" -def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: str | None = None) -> object: +def _snapshot( + cost_map: dict[str, object], backup: str | None = None, schema: str | None = None, pins: str = "" +) -> object: text = _serialize(cost_map) rendered = schema_module.render(schema_module.build_schema(cost_map)) return guard.Snapshot( - cost_map=text, backup=text if backup is None else backup, schema=rendered if schema is None else schema + cost_map=text, + backup=text if backup is None else backup, + schema=rendered if schema is None else schema, + pins=pins, ) @@ -129,9 +134,9 @@ def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () -@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) -def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: - failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) +@pytest.mark.parametrize("checked_path", guard.CHECKED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(checked_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, checked_path), bot=False) assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] @@ -175,8 +180,76 @@ def test_bot_may_not_change_special_root_keys() -> None: assert _failures(head) == ("bot PRs may not change fallback_generalizations",) -def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: +PINNED_SOURCE: Final = "a 512x512 probe image was read back over the provider's chat endpoint" +PINS: Final = _serialize( + {"openrouter/a": {"supports_vision": {"value": True, "checked": "2026-09-26", "source": PINNED_SOURCE}}} +) + + +def test_a_pinned_capability_that_holds_passes_for_humans_and_bots() -> None: + head = _snapshot(BASE_MAP, pins=PINS) + assert _failures(head, bot=False) == () + assert _failures(head, bot=True) == () + + +@pytest.mark.parametrize("bot", [False, True]) +def test_lowering_a_pinned_capability_fails_with_its_evidence(bot: bool) -> None: + head = _snapshot({**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, pins=PINS) + assert _failures(head, bot=bot) == ( + f"{guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: {PINNED_SOURCE}); " + f"change {guard.PINS_PATH} with new evidence first", + ) + + +def test_dropping_a_pinned_field_or_model_is_reported() -> None: + no_field = _snapshot({**BASE_MAP, "openrouter/a": _entry()}, pins=PINS) + assert [failure for failure in _failures(no_field, bot=False) if "must stay true" in failure] + no_model = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/a"}, pins=PINS) + assert _failures(no_model, bot=False) == ( + f"{guard.PINS_PATH} pins openrouter/a.supports_vision but {guard.COST_MAP_PATH} has no openrouter/a entry", + ) + + +@pytest.mark.parametrize( + ("pins", "expected"), + [ + ( + "{not json", + f"{guard.PINS_PATH} is not valid JSON: Expecting property name enclosed in double quotes: " + "line 1 column 2 (char 1)", + ), + ( + _serialize({"openrouter/a": {"supports_vision": {"value": True}}}), + f"{guard.PINS_PATH}: openrouter/a.supports_vision needs value, checked, and source", + ), + (_serialize({"openrouter/a": True}), f"{guard.PINS_PATH}: openrouter/a must map field names to pins"), + ], +) +def test_malformed_pins_are_reported(pins: str, expected: str) -> None: + assert _failures(_snapshot(BASE_MAP, pins=pins), bot=False) == (expected,) + + +def test_bot_may_not_touch_the_pins_file() -> None: + head = _snapshot(BASE_MAP, pins=PINS) + changed = (*guard.GUARDED_PATHS, guard.PINS_PATH) + assert _failures(head, changed_files=changed, bot=False) == () + assert _failures(head, changed_files=changed) == ( + f"bot PRs may only change the cost map files, not {guard.PINS_PATH}", + ) + + +def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: + cost_map = json.loads((ROOT / guard.COST_MAP_PATH).read_text()) + pins = (ROOT / guard.PINS_PATH).read_text() + assert json.loads(pins), "the pins file must pin at least one capability" + assert guard.pin_failures(pins, cost_map) == () + + +def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str: text = _serialize(cost_map) + if pins is not None: + (repo / guard.PINS_PATH).parent.mkdir(exist_ok=True) + (repo / guard.PINS_PATH).write_text(pins) (repo / guard.COST_MAP_PATH).write_text(text) (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) @@ -271,3 +344,25 @@ def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: head = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head") assert _run_guard(tmp_path, base, head, BOT_REF).returncode == 1 assert _run_guard(tmp_path, base, head, "litellm_fix_pricing").returncode == 0 + + +@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"]) +def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: str) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS) + head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "sync lowers it") + result: Final = _run_guard(tmp_path, base, head, head_ref) + assert result.returncode == 1, result.stdout + result.stderr + pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: " + assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)] + + +def test_main_checks_a_pins_only_pr_against_the_head_map(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "base") + (tmp_path / guard.PINS_PATH).parent.mkdir(exist_ok=True) + (tmp_path / guard.PINS_PATH).write_text(PINS) + head: Final = _git_commit(tmp_path, "pin a capability the map disagrees with") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines()