mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
e141596204
* chore(lint): raise basedpyright per-rule slack to 50% of baseline The per-rule ceilings in basedpyright-code-budget.json sat at roughly 10% slack over baseline, which several in-flight PRs are already bumping into. Raise the slack on every rule to at least 50% of its baseline so there is ample headroom for a long while, while never lowering any rule that already had more generous slack (e.g. reportReturnType stays at 100). Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> * refactor(lint): collapse type/lint budgets to a single per-rule limit The three non-frontend budget files (ruff-strict, type-discipline, basedpyright-code) tracked a per-rule baseline and slack whose sum was the ceiling. Nothing consumed the split beyond that sum, so this replaces both keys with a single limit equal to the old baseline + slack; the original baselines live in git history if anyone needs them. The gate scripts and the ratchet guard now read limit directly. lint-budget-update no longer re-captures raw counts; it ratchets each rule's limit down by the number of violations this branch cleared since its branch point (the merge-base), so the granted headroom shrinks by exactly what was fixed and a limit never rises. The ratchet guard reads either schema so it still compares correctly across the migration boundary. Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> * chore(lint): surface staged-vs-working parity for pre-commit and budget-update make pre-commit selects which checks to run from the staged index but runs the linters over the working tree, so unstaged edits to tracked files and untracked files skew a green/red away from what a commit of only the staged changes would produce. There is no safe in-place way to lint the index, so the script now warns when unstaged or untracked changes are present, and CLAUDE.md documents that you must stage everything first for both make pre-commit and make lint-budget-update to predict CI correctly. Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> * docs(lint): list type-discipline budget in lint-budget-update instruction --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
114 lines
4.4 KiB
Python
114 lines
4.4 KiB
Python
"""Tests for scripts/budget_ratchet_check.py.
|
|
|
|
The guard's contract is "limits may only fall": a raised limit, a dropped rule, or
|
|
a deleted file is a regression, while a lowered/equal limit, a brand-new rule, or a
|
|
brand-new budget file is fine. Each branch is pinned here.
|
|
"""
|
|
|
|
import importlib.util
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_MODULE_PATH = (
|
|
Path(__file__).resolve().parents[2] / "scripts" / "budget_ratchet_check.py"
|
|
)
|
|
_spec = importlib.util.spec_from_file_location("budget_ratchet_check", _MODULE_PATH)
|
|
ratchet = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(ratchet)
|
|
|
|
|
|
def _spec_of(limit):
|
|
return {"limit": limit}
|
|
|
|
|
|
def test_limits_read_the_limit_and_skip_malformed():
|
|
limits = ratchet._limits({"LIT006": _spec_of(1023), "junk": 5})
|
|
assert limits == {"LIT006": 1023} # malformed (non-dict) spec ignored
|
|
|
|
|
|
def test_limits_fall_back_to_legacy_baseline_plus_slack():
|
|
# The base side of a diff can predate the `limit` migration; its ceiling is
|
|
# baseline + slack, read on the same footing as a new-schema `limit`.
|
|
assert ratchet._limits({"LIT006": {"baseline": 1013, "slack": 10}}) == {"LIT006": 1023}
|
|
|
|
|
|
def test_migration_from_legacy_schema_to_equal_limit_is_clean():
|
|
# baseline+slack (1023) -> limit 1023 is the same ceiling, so no regression.
|
|
base = {"LIT006": {"baseline": 1013, "slack": 10}}
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == []
|
|
# ...and a genuine raise across the migration is still caught.
|
|
regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1024)})
|
|
assert [r.rule for r in regs] == ["LIT006"] and "1023 -> 1024" in regs[0].detail
|
|
|
|
|
|
def test_raised_limit_is_a_regression():
|
|
base = {"LIT006": _spec_of(1023)}
|
|
head = {"LIT006": _spec_of(1024)}
|
|
regs = ratchet.regressions_for("b.json", base, head)
|
|
assert [r.rule for r in regs] == ["LIT006"]
|
|
assert "1023 -> 1024" in regs[0].detail
|
|
|
|
|
|
def test_lowered_or_equal_limit_is_clean():
|
|
base = {"LIT006": _spec_of(1023)}
|
|
# limit drops
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1000)}) == []
|
|
# nothing changes
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == []
|
|
|
|
|
|
def test_dropped_rule_is_a_regression():
|
|
regs = ratchet.regressions_for("b.json", {"LIT007": _spec_of(0)}, {})
|
|
assert [r.rule for r in regs] == ["LIT007"]
|
|
assert "dropped" in regs[0].detail
|
|
|
|
|
|
def test_new_rule_in_head_is_clean():
|
|
assert ratchet.regressions_for("b.json", {}, {"new-rule": _spec_of(5)}) == []
|
|
|
|
|
|
def test_deleted_budget_file_is_a_regression():
|
|
regs = ratchet.regressions_for("b.json", {"LIT006": _spec_of(1)}, None)
|
|
assert [r.rule for r in regs] == ["*"]
|
|
assert "deleted" in regs[0].detail
|
|
|
|
|
|
def test_new_budget_file_has_nothing_to_ratchet():
|
|
assert ratchet.regressions_for("b.json", None, {"LIT006": _spec_of(1)}) == []
|
|
|
|
|
|
def test_default_budgets_watch_every_budget_file_in_the_repo():
|
|
# This job is the repo's only ceiling-raise alarm, so every *-budget.json on disk must be
|
|
# watched; a budget left out of DEFAULT_BUDGETS (e.g. basedpyright-code-budget.json) can be
|
|
# loosened with no signal. Equality also catches a phantom entry that no longer exists.
|
|
repo_root = _MODULE_PATH.parents[1]
|
|
on_disk = frozenset(p.name for p in repo_root.glob("*budget*.json"))
|
|
assert on_disk == frozenset(ratchet.DEFAULT_BUDGETS)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Base-ref resolution: a bad ref must fail loudly, never pass vacuously
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_ref_is_commit_distinguishes_real_from_bogus():
|
|
assert ratchet._ref_is_commit("HEAD") is True
|
|
assert ratchet._ref_is_commit("definitely-not-a-real-ref-zzz") is False
|
|
|
|
|
|
def test_load_base_reads_a_present_file_and_none_for_an_absent_one():
|
|
# A real budget file exists at HEAD; a made-up path is absent at the same (valid) ref.
|
|
assert ratchet._load_base("type-discipline-budget.json", "HEAD") is not None
|
|
assert ratchet._load_base("scripts/no-such-budget-xyz.json", "HEAD") is None
|
|
|
|
|
|
def test_unresolvable_base_ref_exits_nonzero_instead_of_skipping():
|
|
proc = subprocess.run(
|
|
[sys.executable, str(_MODULE_PATH), "--base", "definitely-not-a-real-ref-zzz"],
|
|
cwd=_MODULE_PATH.parents[1],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert proc.returncode == 1
|
|
assert "does not resolve to a commit" in proc.stderr
|