#!/usr/bin/env python3 """Non-gating ratchet guard: budget limits may only fall, never rise. Every `*-budget.json` file (ruff-strict, type-discipline, basedpyright-code) is a one-way ratchet: each rule's ceiling is its `limit`, and that limit is meant to be driven DOWN over time. This check compares every budget file against its own content at the merge-base with the target branch and fails (exits 1, red) if: * a rule's `limit` went up, * a rule was dropped from a budget (its ceiling effectively became infinite), or * an entire budget file was deleted. New rules and lowered/equal limits are fine. So is a rule that graduated: once a paired config (ruff.toml for the ruff-strict budget) selects the rule outright it hard-fails at the first violation, which is stricter than any ceiling the budget could hold, so dropping its entry tightens the guard rather than removing it. This is deliberately NOT a gating check. It should turn the run red so that a loosening is impossible to miss in review, but it must stay OUT of the branch-protection required-checks list: a justified bump (e.g. banning a new API, which mechanically raises a baseline) can then still be merged by a human who has seen the red and accepted it. Usage: python scripts/budget_ratchet_check.py [--base REF] [budget.json ...] Stdlib only. """ from __future__ import annotations import argparse import json import subprocess import sys import tomllib from pathlib import Path from types import MappingProxyType from typing import NamedTuple REPO_ROOT = Path(__file__).resolve().parent.parent DEFAULT_BASE = "origin/litellm_internal_staging" DEFAULT_BUDGETS: tuple[str, ...] = ( "ruff-strict-budget.json", "type-discipline-budget.json", "basedpyright-code-budget.json", "test-quality-budget.json", ) GRADUATION_CONFIGS = MappingProxyType({"ruff-strict-budget.json": "ruff.toml"}) class Regression(NamedTuple): budget: str rule: str detail: str def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: return subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True) def _merge_base(base: str) -> str: """The common ancestor of `base` and HEAD, so unrelated base drift is ignored.""" proc = _run(["git", "merge-base", base, "HEAD"]) return proc.stdout.strip() or base def _load_head(rel: str) -> dict | None: path = REPO_ROOT / rel if not path.exists(): return None return json.loads(path.read_text()) def _ref_is_commit(ref: str) -> bool: return ( _run( ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"] ).returncode == 0 ) def _load_base(rel: str, ref: str) -> dict | None: """Budget content at `ref`, or None when the file did not exist there. `ref` is verified as a real commit by the caller, so a non-zero `git show` here means the path was absent at that commit, not that the ref itself is unresolvable. """ proc = _run(["git", "show", f"{ref}:{rel}"]) if proc.returncode != 0: return None return json.loads(proc.stdout) def _ceiling(spec: dict) -> int: """A rule's ceiling: its `limit`, or legacy `baseline + slack`. The base side of the diff can predate the `limit` migration, so a spec is read under either schema and the two are compared on the same footing. """ if "limit" in spec: return int(spec["limit"]) return int(spec.get("baseline", 0)) + int(spec.get("slack", 0)) def _limits(budget: dict) -> dict[str, int]: """Map each rule to its ceiling; skip malformed specs.""" return { rule: _ceiling(spec) for rule, spec in budget.items() if isinstance(spec, dict) } def selectors_hard_failed_by(lint: dict) -> tuple[str, ...]: """A ruff `[lint]` table's selected codes, minus anything `ignore` turns back off. `lint.ignore` wins over `lint.extend-select` in ruff, so an ignored code is not actually enforced and must not count as a graduation. """ ignored = tuple(lint.get("ignore", ())) return tuple( selector for selector in lint.get("extend-select", ()) if not (ignored and selector.startswith(ignored)) ) def graduated_selectors(rel: str) -> tuple[str, ...]: """Selectors the budget's paired ruff config hard-fails, so its ceiling is moot.""" config = GRADUATION_CONFIGS.get(rel) if config is None or not (REPO_ROOT / config).exists(): return () return selectors_hard_failed_by( tomllib.loads((REPO_ROOT / config).read_text()).get("lint", {}) ) def _regression_detail( rule: str, base_limits: dict[str, int], head_limits: dict[str, int], graduated: tuple[str, ...], ) -> str | None: """Why `rule` regressed vs base, or None when it held flat, fell, or graduated. A dropped rule is terminal unless it graduated; otherwise the only loosening left is a raised limit. """ base_limit = base_limits[rule] if rule not in head_limits: if graduated and rule.startswith(graduated): return None return f"rule dropped (limit {base_limit} -> removed)" if head_limits[rule] > base_limit: return f"limit raised {base_limit} -> {head_limits[rule]}" return None def regressions_for( rel: str, base: dict | None, head: dict | None, graduated: tuple[str, ...] = (), ) -> list[Regression]: if base is None: return [] # new budget file: nothing to ratchet against yet if head is None: return [Regression(rel, "*", "budget file was deleted (every limit removed)")] base_limits, head_limits = _limits(base), _limits(head) return [ Regression(rel, rule, detail) for rule in sorted(base_limits) if (detail := _regression_detail(rule, base_limits, head_limits, graduated)) is not None ] def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("budgets", nargs="*", help="budget files to check") args = parser.parse_args() budgets = args.budgets or list(DEFAULT_BUDGETS) ref = _merge_base(args.base) if not _ref_is_commit(ref): print( f"FAIL: base ref {ref!r} does not resolve to a commit, so the ratchet has nothing " f"to compare against; refusing to pass vacuously (check the --base / BASE_SHA value)", file=sys.stderr, ) return 1 regressions: list[Regression] = [] checked: list[str] = [] for rel in budgets: base = _load_base(rel, ref) head = _load_head(rel) if base is None and head is None: continue if base is None: print(f"skip {rel}: new file (no base at {args.base} to ratchet against)") continue checked.append(rel) regressions.extend(regressions_for(rel, base, head, graduated_selectors(rel))) if regressions: print( f"FAIL: budget limit(s) loosened vs base {args.base} (merge-base {ref[:12]}):" ) for reg in regressions: print(f" {reg.budget} {reg.rule}: {reg.detail}") print( "Budgets are one-way ratchets and may only go down or stay flat. This " "check is non-gating: if the increase is justified (e.g. a newly banned " "API), a human can merge over the red after acknowledging it." ) return 1 suffix = f" ({', '.join(checked)})" if checked else "" print(f"OK: no budget limit increased vs base {args.base}{suffix}") return 0 if __name__ == "__main__": raise SystemExit(main())