mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
92d0788da2
* chore(lint): widen ruff budget slack to 10% of baseline for high-volume ANN rules and PLR0913 Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> * chore(lint): drop PLR0913 from strict gate to roll out rules gradually Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> * fix(lint): ratchet-guard rising baselines even when slack is cut to mask them Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
120 lines
4.9 KiB
Python
120 lines
4.9 KiB
Python
"""Tests for scripts/budget_ratchet_check.py.
|
|
|
|
The guard's contract is "baselines and ceilings may only fall": a raised ceiling, a
|
|
raised baseline (even when slack is cut to keep the ceiling flat), a dropped rule, or
|
|
a deleted file is a regression, while a lowered/equal baseline and ceiling, a brand-new
|
|
rule, or a brand-new budget file is fine. Each branch is pinned here.
|
|
"""
|
|
|
|
import importlib.util
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_MODULE_PATH = (
|
|
Path(__file__).resolve().parents[2] / "scripts" / "budget_ratchet_check.py"
|
|
)
|
|
_spec = importlib.util.spec_from_file_location("budget_ratchet_check", _MODULE_PATH)
|
|
ratchet = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(ratchet)
|
|
|
|
|
|
def _spec_of(baseline, slack):
|
|
return {"baseline": baseline, "slack": slack}
|
|
|
|
|
|
def test_caps_sum_baseline_and_slack_and_skip_malformed():
|
|
caps = ratchet._caps({"LIT006": _spec_of(1013, 10), "junk": 5})
|
|
assert caps == {"LIT006": 1023} # malformed (non-dict) spec ignored
|
|
|
|
|
|
def test_raised_ceiling_is_a_regression():
|
|
base = {"LIT006": _spec_of(1013, 10)}
|
|
head = {"LIT006": _spec_of(1013, 11)} # cap 1023 -> 1024
|
|
regs = ratchet.regressions_for("b.json", base, head)
|
|
assert [r.rule for r in regs] == ["LIT006"]
|
|
assert "1023 -> 1024" in regs[0].detail
|
|
|
|
|
|
def test_lowered_or_equal_ceiling_is_clean():
|
|
base = {"LIT006": _spec_of(1013, 10)}
|
|
# baseline drops, slack flat -> ceiling falls
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1000, 10)}) == []
|
|
# nothing changes
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1013, 10)}) == []
|
|
# slack cut while baseline holds -> ceiling falls, baseline flat
|
|
assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1013, 0)}) == []
|
|
|
|
|
|
def test_raised_baseline_is_a_regression_even_when_ceiling_held_flat():
|
|
# baseline 1013 -> 1023 with slack cut 10 -> 0 keeps the ceiling at 1023, but a
|
|
# higher baseline bakes in more accepted debt and must still surface as a regression
|
|
base = {"LIT006": _spec_of(1013, 10)}
|
|
regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023, 0)})
|
|
assert [r.rule for r in regs] == ["LIT006"]
|
|
assert "baseline raised 1013 -> 1023" in regs[0].detail
|
|
assert "ceiling raised" not in regs[0].detail
|
|
|
|
|
|
def test_raised_baseline_and_ceiling_report_both_reasons():
|
|
base = {"LIT006": _spec_of(1013, 10)}
|
|
regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1100, 10)})
|
|
assert [r.rule for r in regs] == ["LIT006"]
|
|
assert "ceiling raised 1023 -> 1110" in regs[0].detail
|
|
assert "baseline raised 1013 -> 1100" in regs[0].detail
|
|
|
|
|
|
def test_dropped_rule_is_a_regression():
|
|
regs = ratchet.regressions_for("b.json", {"LIT007": _spec_of(0, 0)}, {})
|
|
assert [r.rule for r in regs] == ["LIT007"]
|
|
assert "dropped" in regs[0].detail
|
|
|
|
|
|
def test_new_rule_in_head_is_clean():
|
|
assert ratchet.regressions_for("b.json", {}, {"new-rule": _spec_of(5, 0)}) == []
|
|
|
|
|
|
def test_deleted_budget_file_is_a_regression():
|
|
regs = ratchet.regressions_for("b.json", {"LIT006": _spec_of(1, 0)}, None)
|
|
assert [r.rule for r in regs] == ["*"]
|
|
assert "deleted" in regs[0].detail
|
|
|
|
|
|
def test_new_budget_file_has_nothing_to_ratchet():
|
|
assert ratchet.regressions_for("b.json", None, {"LIT006": _spec_of(1, 0)}) == []
|
|
|
|
|
|
def test_default_budgets_watch_every_budget_file_in_the_repo():
|
|
# This job is the repo's only ceiling-raise alarm, so every *-budget.json on disk must be
|
|
# watched; a budget left out of DEFAULT_BUDGETS (e.g. basedpyright-code-budget.json) can be
|
|
# loosened with no signal. Equality also catches a phantom entry that no longer exists.
|
|
repo_root = _MODULE_PATH.parents[1]
|
|
on_disk = frozenset(p.name for p in repo_root.glob("*budget*.json"))
|
|
assert on_disk == frozenset(ratchet.DEFAULT_BUDGETS)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Base-ref resolution: a bad ref must fail loudly, never pass vacuously
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_ref_is_commit_distinguishes_real_from_bogus():
|
|
assert ratchet._ref_is_commit("HEAD") is True
|
|
assert ratchet._ref_is_commit("definitely-not-a-real-ref-zzz") is False
|
|
|
|
|
|
def test_load_base_reads_a_present_file_and_none_for_an_absent_one():
|
|
# A real budget file exists at HEAD; a made-up path is absent at the same (valid) ref.
|
|
assert ratchet._load_base("type-discipline-budget.json", "HEAD") is not None
|
|
assert ratchet._load_base("scripts/no-such-budget-xyz.json", "HEAD") is None
|
|
|
|
|
|
def test_unresolvable_base_ref_exits_nonzero_instead_of_skipping():
|
|
proc = subprocess.run(
|
|
[sys.executable, str(_MODULE_PATH), "--base", "definitely-not-a-real-ref-zzz"],
|
|
cwd=_MODULE_PATH.parents[1],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert proc.returncode == 1
|
|
assert "does not resolve to a commit" in proc.stderr
|