diff --git a/.github/workflows/nightly_tighten_basedpyright_budget.yml b/.github/workflows/nightly_tighten_basedpyright_budget.yml new file mode 100644 index 00000000000..b1ac891aae1 --- /dev/null +++ b/.github/workflows/nightly_tighten_basedpyright_budget.yml @@ -0,0 +1,71 @@ +name: Nightly Basedpyright Budget Tighten + +on: + schedule: + - cron: "47 7 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + tighten: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: litellm_internal_staging + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: | + uv sync --frozen --group proxy-dev --group e2e-dev + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Tighten basedpyright budget + env: + NODE_OPTIONS: --max-old-space-size=12288 + run: | + (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --tighten + + - name: Create pull request + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if git diff --quiet basedpyright-code-budget.json; then + echo "No stale headroom to remove; skipping PR." + exit 0 + fi + BRANCH_NAME="litellm_tighten_basedpyright_budget_$(date +'%m_%d_%Y')" + if git ls-remote --exit-code --heads origin "$BRANCH_NAME" > /dev/null; then + echo "Branch $BRANCH_NAME already exists; skipping PR." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH_NAME" + git add basedpyright-code-budget.json + git commit -m "chore(typing): tighten basedpyright budget to staging head counts" + git push "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" "HEAD:refs/heads/$BRANCH_NAME" + gh pr create --base litellm_internal_staging --head "$BRANCH_NAME" \ + --title "chore(typing): tighten basedpyright budget to staging head counts" \ + --body "Automated nightly run of scripts/type_check_gate.py --tighten. Every limit in basedpyright-code-budget.json falls to the current staging head count for its rule plus 50 slack; limits that are already at or below that stay put, so limits only ever fall. The delta-vs-base gate still spares any PR that does not grow a rule past its base, so this only stops new errors from hiding under stale headroom" diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 28602fc235f..390af516b0e 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,15 +1,15 @@ { "reportAny": { - "limit": 34906 + "limit": 24479 }, "reportArgumentType": { - "limit": 2701 + "limit": 2233 }, "reportAssignmentType": { - "limit": 330 + "limit": 280 }, "reportAttributeAccessIssue": { - "limit": 516 + "limit": 422 }, "reportCallIssue": { "limit": 123 @@ -18,19 +18,19 @@ "limit": 59 }, "reportDeprecated": { - "limit": 326 + "limit": 271 }, "reportDuplicateImport": { "limit": 42 }, "reportExplicitAny": { - "limit": 10230 + "limit": 7334 }, "reportFunctionMemberAccess": { "limit": 11 }, "reportGeneralTypeIssues": { - "limit": 227 + "limit": 205 }, "reportIncompatibleMethodOverride": { "limit": 78 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5893 + "limit": 4021 }, "reportMissingTypeArgument": { - "limit": 15886 + "limit": 11127 }, "reportMissingTypeStubs": { "limit": 41 @@ -72,7 +72,7 @@ "limit": 0 }, "reportOptionalMemberAccess": { - "limit": 1085 + "limit": 780 }, "reportOptionalOperand": { "limit": 0 @@ -84,13 +84,13 @@ "limit": 77 }, "reportPrivateUsage": { - "limit": 2438 + "limit": 1870 }, "reportRedeclaration": { "limit": 12 }, "reportReturnType": { - "limit": 225 + "limit": 203 }, "reportTypedDictNotRequiredAccess": { "limit": 27 @@ -99,34 +99,34 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45870 + "limit": 34418 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40525 + "limit": 28432 }, "reportUnknownParameterType": { - "limit": 20384 + "limit": 13955 }, "reportUnknownVariableType": { - "limit": 32099 + "limit": 22373 }, "reportUnnecessaryCast": { - "limit": 177 + "limit": 176 }, "reportUnnecessaryComparison": { - "limit": 1023 + "limit": 760 }, "reportUnnecessaryContains": { "limit": 7 }, "reportUnnecessaryIsInstance": { - "limit": 1206 + "limit": 915 }, "reportUntypedBaseClass": { - "limit": 165 + "limit": 148 }, "reportUntypedFunctionDecorator": { "limit": 33 @@ -135,12 +135,12 @@ "limit": 33 }, "reportUnusedFunction": { - "limit": 206 + "limit": 190 }, "reportUnusedImport": { - "limit": 1005 + "limit": 640 }, "reportUnusedVariable": { - "limit": 1297 + "limit": 198 } } diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 2c5306cec7d..a4f248c3f14 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -305,6 +305,37 @@ def cmd_update(current: Mapping[str, int], base_ref: str = DEFAULT_BASE) -> None ) +def tightened_budget( + budget: Mapping[str, Mapping[str, int]], + current: Mapping[str, int], + slack: int, +) -> dict[str, dict[str, int]]: + return { + code: {"limit": min(spec["limit"], current.get(code, 0) + slack)} + for code, spec in sorted(budget.items()) + } + + +def cmd_tighten( + current: Mapping[str, int], slack: int, budget_path: Path = BUDGET_PATH +) -> None: + budget = json.loads(budget_path.read_text()) + if is_vacuous_run(current, budget): + print( + f"FAIL: basedpyright produced no errors, but {budget_path.name} allows " + f"a nonzero total. The type checker almost certainly crashed or emitted " + f"nothing; refusing to tighten every limit down to the slack floor." + ) + raise SystemExit(1) + updated = tightened_budget(budget, current, slack) + budget_path.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") + removed = sum(budget[code]["limit"] - updated[code]["limit"] for code in updated) + print( + f"Tightened basedpyright limits to head counts plus {slack} slack, " + f"removing {removed} errors of stale headroom across {len(updated)} rules" + ) + + def cmd_check(base_ref: str) -> None: budget = json.loads(BUDGET_PATH.read_text()) head = count_basedpyright(sys.stdin.read()) @@ -353,10 +384,17 @@ def cmd_check(base_ref: str) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", default=DEFAULT_BASE) - parser.add_argument("--update", action="store_true") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--update", action="store_true") + mode.add_argument("--tighten", action="store_true") + parser.add_argument("--slack", type=int, default=50) args = parser.parse_args() + if args.slack < 0: + parser.error("--slack must be >= 0") if args.update: cmd_update(count_basedpyright(sys.stdin.read()), args.base) + elif args.tighten: + cmd_tighten(count_basedpyright(sys.stdin.read()), args.slack) else: cmd_check(args.base) diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index 66a28360af9..95668dbb169 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -287,3 +287,51 @@ def test_an_empty_base_pass_is_never_cached(tmp_path): assert gate.base_counts_cached("abc123", cache_dir=tmp_path, compute=crashed) == {} assert calls == ["abc123", "abc123"] assert list(tmp_path.iterdir()) == [] + + +def test_tighten_collapses_stale_headroom_to_current_plus_slack(): + budget = {"reportAny": {"limit": 34906}} + assert gate.tightened_budget(budget, {"reportAny": 24427}, 50) == { + "reportAny": {"limit": 24477} + } + + +def test_tighten_never_raises_a_limit(): + budget = {"reportAny": {"limit": 100}} + assert gate.tightened_budget(budget, {"reportAny": 90}, 50) == { + "reportAny": {"limit": 100} + } + + +def test_tighten_floors_a_cleared_rule_at_the_slack(): + budget = {"reportAny": {"limit": 100}, "reportCall": {"limit": 8}} + assert gate.tightened_budget(budget, {"reportAny": 0}, 50) == { + "reportAny": {"limit": 50}, + "reportCall": {"limit": 8}, + } + + +def test_tighten_does_not_adopt_rules_outside_the_budget(): + assert gate.tightened_budget({}, {"brand-new": 7}, 50) == {} + + +def test_tighten_writes_only_lowered_limits(tmp_path): + budget_path = tmp_path / "budget.json" + budget_path.write_text( + json.dumps({"reportAny": {"limit": 100}, "reportCall": {"limit": 3}}) + ) + gate.cmd_tighten({"reportAny": 20, "reportCall": 5}, 10, budget_path=budget_path) + assert json.loads(budget_path.read_text()) == { + "reportAny": {"limit": 30}, + "reportCall": {"limit": 3}, + } + + +def test_tighten_refuses_a_vacuous_run(tmp_path): + import pytest + + budget_path = tmp_path / "budget.json" + budget_path.write_text(json.dumps({"reportAny": {"limit": 100}})) + with pytest.raises(SystemExit): + gate.cmd_tighten({}, 50, budget_path=budget_path) + assert json.loads(budget_path.read_text()) == {"reportAny": {"limit": 100}}