diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index f212dd9d15e..2e967f3ed3f 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -14,11 +14,15 @@ permissions: jobs: lint: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + # Check out the PR head, not the default refs/pull/N/merge: the merge ref + # folds in newer base commits, which the diff-based gates (ruff delta, + # Any-discipline) would otherwise blame on this branch. with: + ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 clean: true persist-credentials: false @@ -80,8 +84,11 @@ jobs: - name: Run MyPy type checking run: | cd litellm - uv run --no-sync mypy . - cd .. + (uv run --no-sync mypy . || true) | uv run --no-sync python ../scripts/type_check_gate.py --tool mypy + + - name: Run basedpyright type checking + run: | + (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --tool basedpyright - name: Check for circular imports run: | @@ -93,6 +100,56 @@ jobs: run: | uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + any-discipline: + # Separate job: the first run cold-builds litellm's type cache (~2 min, ~3 GB), + # so keep it off the main lint job's time budget. Subsequent runs reuse the + # cached .mypy_cache_any and only re-type-check the changed files. + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + # Check out the PR head, not the default refs/pull/N/merge: the merge ref + # folds in newer base commits, which the diff-based gates (ruff delta, + # Any-discipline) would otherwise blame on this branch. + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + clean: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Install dependencies + run: | + uv sync --frozen + + # Keyed on deps + mypy config (which fix the type cache's validity), not on + # source content, so changed files always differ from the restored cache. + # The gate also defensively invalidates each target's cache entry, so + # correctness never depends on cache freshness -- this is purely for speed. + - name: Restore Any-gate type cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .mypy_cache_any + key: any-mypy-cache-${{ runner.os }}-py3.12-${{ hashFiles('uv.lock', 'litellm/mypy.ini') }} + restore-keys: | + any-mypy-cache-${{ runner.os }}-py3.12- + + - name: Check Any discipline on changed lines + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + uv run --no-sync python scripts/check_any_discipline.py --changed --base "$BASE_SHA" + secret-scan: runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.gitignore b/.gitignore index 572830d35f6..54ae53bb2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ tests/local_testing/log.txt litellm/proxy/_new_new_secret_config.yaml litellm/proxy/custom_guardrail.py **/.mypy_cache/ +**/.mypy_cache_any/ litellm/proxy/application.log tests/llm_translation/vertex_test_account.json tests/llm_translation/test_vertex_key.json diff --git a/CLAUDE.md b/CLAUDE.md index 32fd0aadddb..48dc3d81d94 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,11 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a Run tests, format your code, and lint your code before each commit -When you fix strict-rule violations gated by `ruff-strict-budget.json`, run `make lint-strict-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom +When you fix violations gated by `ruff-strict-budget.json`, `mypy-code-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom + +If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in + +The Any-discipline gate (`make lint-any`, also a CI job) fails when a line you changed under `litellm/` holds a value typed `Any`, including the `X | Any`. Ideally `# any-ok: ` is never used; treat it as a last resort for a genuine typed/untyped boundary that Pydantic truly can't model Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it) @@ -67,8 +71,6 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions -if you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller (a simple function that returns the typed thing or raises will do) and then pass the now typed variable in - Follow conventional commits for commit names and PR titles ## Think Before Coding diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2177c764806..97a8d53f831 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,7 @@ Individual linting commands: make format-check # Check Black formatting make lint-ruff # Run Ruff linting make lint-mypy # Run MyPy type checking +make lint-any # Fail on Any-typed values on changed lines make check-circular-imports # Check for circular imports make check-import-safety # Check import safety ``` diff --git a/Makefile b/Makefile index fe4be29f9b6..f0563b273c2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,8 @@ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ - lint-strict-budget lint-strict-budget-update \ + lint-mypy lint-mypy-budget-update lint-basedpyright lint-basedpyright-budget-update \ + lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-any \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety @@ -23,10 +24,15 @@ help: @echo " make format-check - Check Black code formatting (matches CI)" @echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)" @echo " make lint-ruff - Run Ruff linting only" - @echo " make lint-mypy - Run MyPy type checking only" + @echo " make lint-mypy - Run MyPy (disallow_untyped_defs), gated by per-rule error counts" + @echo " make lint-mypy-budget-update - Re-capture the MyPy per-rule budget (ratchet)" + @echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts" + @echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)" @echo " make lint-black - Check Black formatting (matches CI)" - @echo " make lint-strict-budget - Gate the codebase total of each strict ruff rule against its ceiling" - @echo " make lint-strict-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)" + @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling" + @echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)" + @echo " make lint-budget-update - Re-capture all three ratchet budgets (ruff + mypy + basedpyright)" + @echo " make lint-any - Fail if changed lines under litellm/ hold an Any-typed value" @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @@ -121,16 +127,31 @@ lint-ruff-FULL-dev: install-dev else echo "No changed .py files to check."; fi lint-mypy: install-dev - cd litellm && $(UV_RUN) mypy . --ignore-missing-imports && cd .. + cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy + +lint-mypy-budget-update: install-dev + cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy --update + +lint-basedpyright: install-dev + ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright + +lint-basedpyright-budget-update: install-dev + ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright --update lint-black: format-check -lint-strict-budget: install-dev +lint-ruff-budget: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py -lint-strict-budget-update: install-dev +lint-ruff-budget-update: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py --update +# Ratchet all three budgets in one shot (ruff strict + mypy + basedpyright) +lint-budget-update: lint-ruff-budget-update lint-mypy-budget-update lint-basedpyright-budget-update + +lint-any: install-dev + $(UV_RUN) python scripts/check_any_discipline.py --changed + check-circular-imports: install-dev cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd .. @@ -138,10 +159,10 @@ check-import-safety: install-dev @$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) # Combined linting (matches test-linting.yml workflow) -lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety lint-strict-budget +lint: format-check lint-ruff lint-mypy lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget lint-any # Faster linting for local development (only checks changed code) -lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety +lint-dev: lint-format-changed lint-mypy lint-any check-circular-imports check-import-safety # Testing targets test: install-test-deps diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json new file mode 100644 index 00000000000..b531e0e17df --- /dev/null +++ b/basedpyright-code-budget.json @@ -0,0 +1,194 @@ +{ + "reportAny": { + "baseline": 24954, + "slack": 10 + }, + "reportArgumentType": { + "baseline": 1863, + "slack": 3 + }, + "reportAssignmentType": { + "baseline": 220, + "slack": 3 + }, + "reportAttributeAccessIssue": { + "baseline": 335, + "slack": 3 + }, + "reportCallIssue": { + "baseline": 77, + "slack": 10 + }, + "reportConstantRedefinition": { + "baseline": 39, + "slack": 3 + }, + "reportDeprecated": { + "baseline": 217, + "slack": 10 + }, + "reportDuplicateImport": { + "baseline": 28, + "slack": 3 + }, + "reportExplicitAny": { + "baseline": 6931, + "slack": 10 + }, + "reportFunctionMemberAccess": { + "baseline": 7, + "slack": 3 + }, + "reportGeneralTypeIssues": { + "baseline": 151, + "slack": 3 + }, + "reportIncompatibleMethodOverride": { + "baseline": 52, + "slack": 10 + }, + "reportIncompatibleVariableOverride": { + "baseline": 8, + "slack": 3 + }, + "reportInconsistentOverload": { + "baseline": 12, + "slack": 3 + }, + "reportIndexIssue": { + "baseline": 26, + "slack": 3 + }, + "reportInvalidTypeForm": { + "baseline": 23, + "slack": 3 + }, + "reportInvalidTypeVarUse": { + "baseline": 2, + "slack": 3 + }, + "reportMatchNotExhaustive": { + "baseline": 1, + "slack": 3 + }, + "reportMissingParameterType": { + "baseline": 3933, + "slack": 10 + }, + "reportMissingTypeArgument": { + "baseline": 10612, + "slack": 10 + }, + "reportMissingTypeStubs": { + "baseline": 27, + "slack": 10 + }, + "reportOperatorIssue": { + "baseline": 6, + "slack": 3 + }, + "reportOptionalCall": { + "baseline": 4, + "slack": 3 + }, + "reportOptionalIterable": { + "baseline": 3, + "slack": 3 + }, + "reportOptionalMemberAccess": { + "baseline": 724, + "slack": 10 + }, + "reportOptionalOperand": { + "baseline": 3, + "slack": 3 + }, + "reportOptionalSubscript": { + "baseline": 11, + "slack": 3 + }, + "reportPossiblyUnboundVariable": { + "baseline": 52, + "slack": 10 + }, + "reportPrivateUsage": { + "baseline": 1625, + "slack": 10 + }, + "reportRedeclaration": { + "baseline": 8, + "slack": 3 + }, + "reportReturnType": { + "baseline": 118, + "slack": 10 + }, + "reportTypedDictNotRequiredAccess": { + "baseline": 20, + "slack": 3 + }, + "reportUndefinedVariable": { + "baseline": 2, + "slack": 3 + }, + "reportUnknownArgumentType": { + "baseline": 30603, + "slack": 10 + }, + "reportUnknownLambdaType": { + "baseline": 76, + "slack": 10 + }, + "reportUnknownMemberType": { + "baseline": 27322, + "slack": 10 + }, + "reportUnknownParameterType": { + "baseline": 13636, + "slack": 10 + }, + "reportUnknownVariableType": { + "baseline": 21776, + "slack": 10 + }, + "reportUnnecessaryCast": { + "baseline": 118, + "slack": 10 + }, + "reportUnnecessaryComparison": { + "baseline": 680, + "slack": 10 + }, + "reportUnnecessaryContains": { + "baseline": 4, + "slack": 3 + }, + "reportUnnecessaryIsInstance": { + "baseline": 807, + "slack": 10 + }, + "reportUntypedBaseClass": { + "baseline": 110, + "slack": 3 + }, + "reportUntypedFunctionDecorator": { + "baseline": 22, + "slack": 3 + }, + "reportUnusedClass": { + "baseline": 22, + "slack": 3 + }, + "reportUnusedFunction": { + "baseline": 137, + "slack": 10 + }, + "reportUnusedImport": { + "baseline": 670, + "slack": 10 + }, + "reportUnusedVariable": { + "baseline": 865, + "slack": 10 + } +} diff --git a/litellm/mypy.ini b/litellm/mypy.ini index 4702b591124..b65e11bab42 100644 --- a/litellm/mypy.ini +++ b/litellm/mypy.ini @@ -1,13 +1,16 @@ [mypy] -warn_return_any = False +warn_return_any = True ignore_missing_imports = True +disallow_untyped_defs = True mypy_path = litellm/stubs namespace_packages = True disable_error_code = - valid-type, annotation-unchecked, import-untyped +[mypy-litellm.*] +ignore_missing_imports = False + [mypy-google.*] ignore_missing_imports = True diff --git a/mypy-code-budget.json b/mypy-code-budget.json new file mode 100644 index 00000000000..2cae0d661e9 --- /dev/null +++ b/mypy-code-budget.json @@ -0,0 +1,18 @@ +{ + "import-not-found": { + "baseline": 8, + "slack": 3 + }, + "no-any-return": { + "baseline": 902, + "slack": 10 + }, + "no-untyped-def": { + "baseline": 4888, + "slack": 10 + }, + "valid-type": { + "baseline": 1, + "slack": 3 + } +} diff --git a/pyproject.toml b/pyproject.toml index 6429b810969..8b1386aaf87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,7 @@ dev = [ "flake8==7.3.0", "black==26.3.1", "mypy==1.19.0", + "basedpyright==1.39.7", "pytest==9.0.3", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", @@ -220,7 +221,6 @@ ci = [ "blockbuster==1.5.26", "beautifulsoup4==4.14.3", "pylint==4.0.5", - "pyright==1.1.408", "langchain-mcp-adapters==0.2.1", "langchain-openai==1.1.14", "langgraph==1.0.10", diff --git a/pyrightconfig.json b/pyrightconfig.json index f930e44d305..97f099d5b2c 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,7 +1,12 @@ { + "include": ["litellm"], "ignore": [], "exclude": ["**/node_modules", "**/__pycache__", "litellm/types/utils.py", "litellm/proxy/_types.py"], + "pythonVersion": "3.12", + "typeCheckingMode": "strict", + "enableTypeIgnoreComments": false, "reportMissingImports": false, - "reportPrivateImportUsage": false + "reportPrivateImportUsage": false, + "reportExplicitAny": "error", + "reportAny": "error" } - \ No newline at end of file diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 6363b72353f..bb02ec01569 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,12 +1,494 @@ { - "ANN001": { "baseline": 2865, "slack": 10 }, - "ANN002": { "baseline": 64, "slack": 3 }, - "ANN003": { "baseline": 759, "slack": 10 }, - "ANN401": { "baseline": 1885, "slack": 10 }, - "B006": { "baseline": 180, "slack": 3 }, - "C901": { "baseline": 301, "slack": 3 }, - "PLR0913": { "baseline": 1813, "slack": 3 }, - "PLW0603": { "baseline": 183, "slack": 3 }, - "RUF012": { "baseline": 158, "slack": 3 }, - "TID251": { "baseline": 2404, "slack": 10 } + "ANN001": { + "baseline": 2865, + "slack": 10 + }, + "ANN002": { + "baseline": 64, + "slack": 3 + }, + "ANN003": { + "baseline": 759, + "slack": 10 + }, + "ANN201": { + "baseline": 1944, + "slack": 10 + }, + "ANN202": { + "baseline": 858, + "slack": 10 + }, + "ANN204": { + "baseline": 658, + "slack": 10 + }, + "ANN205": { + "baseline": 117, + "slack": 10 + }, + "ANN206": { + "baseline": 120, + "slack": 10 + }, + "ANN401": { + "baseline": 1886, + "slack": 10 + }, + "ASYNC230": { + "baseline": 11, + "slack": 3 + }, + "B004": { + "baseline": 1, + "slack": 3 + }, + "B006": { + "baseline": 180, + "slack": 3 + }, + "B008": { + "baseline": 490, + "slack": 10 + }, + "B009": { + "baseline": 79, + "slack": 10 + }, + "B010": { + "baseline": 187, + "slack": 10 + }, + "B018": { + "baseline": 2, + "slack": 3 + }, + "B019": { + "baseline": 1, + "slack": 3 + }, + "B021": { + "baseline": 1, + "slack": 3 + }, + "B026": { + "baseline": 3, + "slack": 3 + }, + "B033": { + "baseline": 1, + "slack": 3 + }, + "BLE001": { + "baseline": 2854, + "slack": 10 + }, + "C401": { + "baseline": 8, + "slack": 3 + }, + "C404": { + "baseline": 1, + "slack": 3 + }, + "C405": { + "baseline": 20, + "slack": 3 + }, + "C408": { + "baseline": 11, + "slack": 3 + }, + "C414": { + "baseline": 4, + "slack": 3 + }, + "C419": { + "baseline": 1, + "slack": 3 + }, + "C901": { + "baseline": 301, + "slack": 3 + }, + "D419": { + "baseline": 6, + "slack": 3 + }, + "DTZ001": { + "baseline": 2, + "slack": 3 + }, + "DTZ003": { + "baseline": 30, + "slack": 3 + }, + "DTZ005": { + "baseline": 229, + "slack": 10 + }, + "DTZ006": { + "baseline": 10, + "slack": 3 + }, + "DTZ007": { + "baseline": 20, + "slack": 3 + }, + "DTZ011": { + "baseline": 3, + "slack": 3 + }, + "EXE001": { + "baseline": 4, + "slack": 3 + }, + "EXE002": { + "baseline": 3, + "slack": 3 + }, + "F401": { + "baseline": 20, + "slack": 3 + }, + "FURB136": { + "baseline": 1, + "slack": 3 + }, + "FURB168": { + "baseline": 1, + "slack": 3 + }, + "FURB188": { + "baseline": 49, + "slack": 3 + }, + "I001": { + "baseline": 258, + "slack": 10 + }, + "LOG015": { + "baseline": 5, + "slack": 3 + }, + "N999": { + "baseline": 1, + "slack": 3 + }, + "PERF102": { + "baseline": 27, + "slack": 3 + }, + "PERF401": { + "baseline": 136, + "slack": 10 + }, + "PERF402": { + "baseline": 6, + "slack": 3 + }, + "PERF403": { + "baseline": 69, + "slack": 10 + }, + "PIE790": { + "baseline": 263, + "slack": 10 + }, + "PIE800": { + "baseline": 1, + "slack": 3 + }, + "PIE804": { + "baseline": 21, + "slack": 3 + }, + "PIE810": { + "baseline": 41, + "slack": 3 + }, + "PLC0206": { + "baseline": 28, + "slack": 3 + }, + "PLC0208": { + "baseline": 1, + "slack": 3 + }, + "PLC0414": { + "baseline": 35, + "slack": 3 + }, + "PLR0124": { + "baseline": 1, + "slack": 3 + }, + "PLR0206": { + "baseline": 1, + "slack": 3 + }, + "PLR0402": { + "baseline": 6, + "slack": 3 + }, + "PLR0913": { + "baseline": 1813, + "slack": 3 + }, + "PLR1704": { + "baseline": 3, + "slack": 3 + }, + "PLR1711": { + "baseline": 31, + "slack": 3 + }, + "PLR1714": { + "baseline": 252, + "slack": 10 + }, + "PLR1730": { + "baseline": 7, + "slack": 3 + }, + "PLR2044": { + "baseline": 1, + "slack": 3 + }, + "PLW0127": { + "baseline": 41, + "slack": 3 + }, + "PLW0133": { + "baseline": 1, + "slack": 3 + }, + "PLW0602": { + "baseline": 215, + "slack": 10 + }, + "PLW0603": { + "baseline": 183, + "slack": 3 + }, + "PLW1508": { + "baseline": 188, + "slack": 10 + }, + "PLW1510": { + "baseline": 2, + "slack": 3 + }, + "PYI030": { + "baseline": 2, + "slack": 3 + }, + "PYI036": { + "baseline": 2, + "slack": 3 + }, + "PYI041": { + "baseline": 9, + "slack": 3 + }, + "PYI064": { + "baseline": 2, + "slack": 3 + }, + "RET501": { + "baseline": 35, + "slack": 3 + }, + "RET504": { + "baseline": 709, + "slack": 10 + }, + "RUF010": { + "baseline": 844, + "slack": 10 + }, + "RUF012": { + "baseline": 158, + "slack": 3 + }, + "RUF015": { + "baseline": 8, + "slack": 3 + }, + "RUF019": { + "baseline": 38, + "slack": 3 + }, + "RUF022": { + "baseline": 80, + "slack": 10 + }, + "RUF023": { + "baseline": 2, + "slack": 3 + }, + "RUF046": { + "baseline": 5, + "slack": 3 + }, + "RUF051": { + "baseline": 3, + "slack": 3 + }, + "RUF059": { + "baseline": 69, + "slack": 10 + }, + "RUF100": { + "baseline": 465, + "slack": 10 + }, + "S110": { + "baseline": 222, + "slack": 10 + }, + "S112": { + "baseline": 21, + "slack": 3 + }, + "SIM101": { + "baseline": 58, + "slack": 10 + }, + "SIM102": { + "baseline": 311, + "slack": 10 + }, + "SIM103": { + "baseline": 119, + "slack": 10 + }, + "SIM113": { + "baseline": 3, + "slack": 3 + }, + "SIM114": { + "baseline": 103, + "slack": 10 + }, + "SIM115": { + "baseline": 2, + "slack": 3 + }, + "SIM117": { + "baseline": 7, + "slack": 3 + }, + "SIM118": { + "baseline": 104, + "slack": 10 + }, + "SIM201": { + "baseline": 1, + "slack": 3 + }, + "SIM210": { + "baseline": 9, + "slack": 3 + }, + "SIM211": { + "baseline": 1, + "slack": 3 + }, + "SIM222": { + "baseline": 1, + "slack": 3 + }, + "SIM401": { + "baseline": 9, + "slack": 3 + }, + "TC004": { + "baseline": 5, + "slack": 3 + }, + "TC005": { + "baseline": 6, + "slack": 3 + }, + "TID251": { + "baseline": 2405, + "slack": 10 + }, + "TRY002": { + "baseline": 528, + "slack": 10 + }, + "TRY004": { + "baseline": 93, + "slack": 10 + }, + "TRY201": { + "baseline": 409, + "slack": 10 + }, + "TRY203": { + "baseline": 113, + "slack": 10 + }, + "TRY300": { + "baseline": 853, + "slack": 10 + }, + "UP006": { + "baseline": 12941, + "slack": 10 + }, + "UP007": { + "baseline": 2520, + "slack": 10 + }, + "UP008": { + "baseline": 2, + "slack": 3 + }, + "UP012": { + "baseline": 4, + "slack": 3 + }, + "UP018": { + "baseline": 18, + "slack": 3 + }, + "UP024": { + "baseline": 12, + "slack": 3 + }, + "UP028": { + "baseline": 2, + "slack": 3 + }, + "UP031": { + "baseline": 2, + "slack": 3 + }, + "UP032": { + "baseline": 609, + "slack": 10 + }, + "UP034": { + "baseline": 1, + "slack": 3 + }, + "UP035": { + "baseline": 2250, + "slack": 10 + }, + "UP036": { + "baseline": 1, + "slack": 3 + }, + "UP037": { + "baseline": 100, + "slack": 10 + }, + "UP045": { + "baseline": 18417, + "slack": 10 + } } diff --git a/ruff-strict.toml b/ruff-strict.toml index 03145255ebf..8d517615244 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -1,7 +1,8 @@ extend = "ruff.toml" [lint] -select = ["ANN001", "ANN002", "ANN003", "ANN401", "B006", "C901", "PLR0913", "PLW0603", "RUF012", "TID251"] +preview = true +select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR0913", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] extend-select = [] [lint.mccabe] diff --git a/scripts/check_any_discipline.py b/scripts/check_any_discipline.py new file mode 100644 index 00000000000..3185953d473 --- /dev/null +++ b/scripts/check_any_discipline.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""Any-discipline gate: fail when a *changed* file holds a value typed `Any`. + +Where ruff, `mypy --strict`, and even basedpyright's `reportAny` stop short, this +catches the case that actually bites: a *union* hiding an `Any`. For example +`re.Match.group()` -> `str | Any`, `json.loads()` -> `Any`, and bare `list`/`dict` +-> `list[Any]`/`dict[..., Any]`. Any value whose inferred type *contains* `Any` +(recursively, through unions / generics / tuples) is reported. + +Scope: changed-only, changed-lines +---------------------------------- +litellm already contains a large amount of pre-existing `Any` (a single legacy +file can have >100 findings), and a whole-tree scan would have to re-export types +for litellm's entire import closure on every run (~2 min, ~3 GB). So this gate is +*changed-only* and reports a finding only on a line that the diff against +`--base` actually adds or edits (untracked files count as wholly new). A brand +new file is therefore checked in full, while editing a legacy file only requires +*your* lines to be clean -- you can't introduce an `X | Any`, but you aren't +forced to clean the file's existing debt. This mirrors how `ruff_strict_gate.py` +blames a change only for the violations it introduces; cold legacy code is left +to the ratchet gates (mypy/basedpyright/ruff budgets). + +How it works +------------ +It loads `litellm/mypy.ini` (the same config `make lint-mypy` uses, so findings +match what developers already see), builds the changed files with mypy asking for +its exported expression->type map, and walks each file's AST applying a recursive +"contains Any" predicate -- the test `mypy --disallow-any-expr` uses internally +but applies inconsistently (python/mypy#12856). + +mypy only re-exports types for modules it re-type-checks, so for each target we +invalidate just its cached hash (deps stay warm) to force a fast re-check against +a persisted incremental cache (.mypy_cache_any). + +Rules +----- +Codes share the `LIT***` namespace with `scripts/check_type_discipline.py` (PR +#30500), which owns LIT001/002/003/004/006/007/008. This gate claims the rest: +LIT009 A value expression's inferred type is, or contains, `Any`. + Suppress with `# any-ok: ` on the offending line. +LIT005 An `# any-ok` suppression without a reason (the shared + suppression-needs-a-reason code, same as `# cast-ok` / `# guard-ok`). +LIT000 Setup failure: mypy could not build, or a target file could not be read. + +`Any`s produced purely by an already-reported error, and the special-form / +implementation-artifact internal `Any`s, are ignored. A bound method *reference* +whose signature mentions `Any` is not flagged -- only the value its call produces. + +Usage +----- + # gate mode (CI / pre-push): check changed lines under litellm/ + uv run --no-sync python scripts/check_any_discipline.py --changed --base origin/litellm_internal_staging + + # whole-file spot-check (no line filter), paths relative to repo root + uv run --no-sync python scripts/check_any_discipline.py litellm/budget_manager.py + +Exit code 1 if any Any-tainted value is found, 2 on a setup/usage error. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tokenize +from collections.abc import Iterable, Sequence +from pathlib import Path +from typing import NamedTuple + +try: + from mypy import build + from mypy.config_parser import parse_config_file + from mypy.find_sources import create_source_list + from mypy.fscache import FileSystemCache + from mypy.modulefinder import BuildSource + from mypy.nodes import AssignmentStmt, Expression, NameExpr, Node + from mypy.options import Options + from mypy.types import ( + AnyType, + CallableType, + Instance, + Overloaded, + TupleType, + Type, + TypeOfAny, + UnionType, + get_proper_type, + ) +except ImportError: # pragma: no cover - environment guard + sys.stderr.write( + "check_any_discipline: mypy is not importable in this interpreter.\n" + "Run it through the project environment, e.g.\n" + " uv run --no-sync python scripts/check_any_discipline.py --changed\n" + ) + raise SystemExit(2) + + +REPO_ROOT = Path(__file__).resolve().parent.parent +LITELLM_DIR = REPO_ROOT / "litellm" +MYPY_INI = LITELLM_DIR / "mypy.ini" +CACHE_DIR = REPO_ROOT / ".mypy_cache_any" +PY_TAG = f"{sys.version_info.major}.{sys.version_info.minor}" +DEFAULT_BASE = "origin/litellm_internal_staging" + +MIN_REASON_LEN = 3 +ANY_OK_RE = re.compile(r"#\s*any-ok(?::\s*(?P.*))?") +_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + +# Files allowed to surface `Any` (the typed/untyped boundary). A finding is +# skipped if any fragment below is a substring of the file's posix path. Keep +# this tight -- prefer a line-level `# any-ok: ` over a blanket exemption. +BOUNDARY_PATHS: frozenset[str] = frozenset() + +# `Any` kinds that are not actionable: produced by an already-reported error, or +# an internal placeholder that never corresponds to a concrete runtime value. +# NOTE: `special_form` is deliberately NOT here. In mypy 1.19 the `Any` in +# typeshed unions like `re.Match.group() -> str | Any` is tagged `special_form`, +# and that union is the headline case this gate exists to catch. +_HARMLESS_ANY = frozenset( + kind + for kind in ( + TypeOfAny.from_error, + getattr(TypeOfAny, "implementation_artifact", None), + ) + if kind is not None +) + +# AST attributes that point OUTSIDE the syntactic subtree (a RefExpr's resolved +# definition, a node's TypeInfo). Skipping exactly these two makes a generic +# child-walk equivalent to mypy's TraverserVisitor -- validated to the node +# against ExtendedTraverserVisitor across the full grammar (see commit notes). +_NON_SYNTACTIC_ATTRS = frozenset({"node", "info"}) + + +class Violation(NamedTuple): + path: Path + line: int + col: int + code: str + message: str + + def render(self) -> str: + return f"{self.path}:{self.line}:{self.col}: {self.code} {self.message}" + + +# --------------------------------------------------------------------------- # +# The "contains Any" predicate +# --------------------------------------------------------------------------- # + + +def contains_any(t: Type, _seen: set[int] | None = None) -> bool: + """True if a *value* of type ``t`` carries `Any` anywhere meaningful.""" + seen = _seen if _seen is not None else set() + p = get_proper_type(t) + if id(p) in seen: + return False + seen.add(id(p)) + + # A function/method *reference* whose signature mentions Any is not itself an + # unsafe value -- only its eventual call result is. Don't recurse into it. + if isinstance(p, (CallableType, Overloaded)): + return False + if isinstance(p, AnyType): + return p.type_of_any not in _HARMLESS_ANY + if isinstance(p, UnionType): + return any(contains_any(item, seen) for item in p.items) + if isinstance(p, Instance): + return any(contains_any(arg, seen) for arg in p.args) + if isinstance(p, TupleType): + return any(contains_any(item, seen) for item in p.items) + return False + + +# --------------------------------------------------------------------------- # +# Generic, leak-free AST walk (works under a mypyc-compiled mypy, which forbids +# subclassing TraverserVisitor) +# --------------------------------------------------------------------------- # + + +def _walk_file(tree: Node) -> tuple[list[Expression], set[int]]: + """Return (every Expression in `tree`, ids of simple assignment-target names). + + The walk follows only syntactic children (every attribute except the two + non-syntactic back-references), so it never escapes the module. Simple + ``x = `` name targets are collected separately so we don't double-report + the assigned name as an echo of an Any rvalue. + """ + exprs: list[Expression] = [] + skip_lvalues: set[int] = set() + stack: list[object] = [tree] + seen: set[int] = set() + while stack: + n = stack.pop() + if isinstance(n, Node): + if id(n) in seen: + continue + seen.add(id(n)) + if isinstance(n, Expression): + exprs.append(n) + if isinstance(n, AssignmentStmt): + for lvalue in n.lvalues: + if isinstance(lvalue, NameExpr): + skip_lvalues.add(id(lvalue)) + for name in dir(n): + if name.startswith("__") or name in _NON_SYNTACTIC_ATTRS: + continue + try: + val = getattr(n, name) + except Exception: + continue + if callable(val): + continue + if isinstance(val, (Node, list, tuple)): + stack.append(val) + elif isinstance(n, (list, tuple)): + stack.extend(n) + return exprs, skip_lvalues + + +def find_any_in_tree(tree: Node, idmap: dict[int, Type]) -> list[tuple[int, int, str]]: + exprs, skip_lvalues = _walk_file(tree) + findings: list[tuple[int, int, str]] = [] + for expr in exprs: + if id(expr) in skip_lvalues: + continue + t = idmap.get(id(expr)) + if t is not None and contains_any(t): + findings.append((expr.line, expr.column, str(get_proper_type(t)))) + + out: list[tuple[int, int, str]] = [] + seen_pos: set[tuple[int, int]] = set() + for line, col, typ in sorted(findings): + if line < 1 or (line, col) in seen_pos: + continue + seen_pos.add((line, col)) + out.append((line, col, typ)) + return out + + +# --------------------------------------------------------------------------- # +# Comment scanning (LIT005 + any-ok suppression) +# --------------------------------------------------------------------------- # + + +def _reason_ok(reason: str | None) -> bool: + return reason is not None and len(reason.strip()) >= MIN_REASON_LEN + + +def scan_any_ok( + path: Path, source: str +) -> tuple[frozenset[int], tuple[Violation, ...]]: + """Return (lines with a valid any-ok suppression, LIT005 violations).""" + try: + tokens = tokenize.generate_tokens( + iter(source.splitlines(keepends=True)).__next__ + ) + comments = tuple( + (t.start[0], t.string) for t in tokens if t.type == tokenize.COMMENT + ) + except tokenize.TokenError: + return frozenset(), () + + ok_lines: set[int] = set() + violations: list[Violation] = [] + for line, text in comments: + m = ANY_OK_RE.search(text) + if m is None: + continue + if _reason_ok(m.group("reason")): + ok_lines.add(line) + else: + violations.append( + Violation( + path, + line, + 0, + "LIT005", + "any-ok requires a reason: `# any-ok: `", + ) + ) + return frozenset(ok_lines), tuple(violations) + + +# --------------------------------------------------------------------------- # +# mypy build (parity with `make lint-mypy`) + forced target re-check +# --------------------------------------------------------------------------- # + + +def _build_options() -> Options: + opts = Options() + if MYPY_INI.exists(): + parse_config_file(opts, lambda: None, str(MYPY_INI), sys.stdout, sys.stderr) + opts.export_types = True + opts.preserve_asts = True + opts.incremental = True + opts.cache_dir = str(CACHE_DIR) + opts.show_traceback = False + return opts + + +def _meta_path(module: str) -> Path: + return CACHE_DIR / PY_TAG / (module.replace(".", os.sep) + ".meta.json") + + +def _force_recheck(sources: Sequence[BuildSource]) -> None: + """Invalidate each target's cached entry so mypy re-type-checks (and thus + re-exports types + preserves the AST for) exactly these modules, while their + dependencies stay warm. A missing entry is a cold build for that module. + + mypy trusts a cache entry whenever the source mtime matches the cached one + (it never re-hashes on that fast path), so we must break BOTH: zero the + cached mtime to force a re-hash, and corrupt the cached hash so the re-hash + mismatches and the module is treated as changed.""" + for src in sources: + if not src.module: + continue + meta = _meta_path(src.module) + if not meta.exists(): + continue + try: + data = json.loads(meta.read_text()) + data["hash"] = "0" * 40 + data["mtime"] = 0 + meta.write_text(json.dumps(data)) + except (OSError, ValueError): + continue + + +def check_files(rel_paths: Sequence[str]) -> tuple[Violation, ...]: + """`rel_paths` are relative to the litellm package dir (the build cwd).""" + prev_cwd = Path.cwd() + os.chdir(LITELLM_DIR) + try: + opts = _build_options() + fscache = FileSystemCache() + sources = create_source_list(list(rel_paths), opts, fscache) + _force_recheck(sources) + try: + res = build.build(sources, options=opts, fscache=fscache) + except build.CompileError as exc: + joined = "; ".join(exc.messages[:3]) or "blocking error" + return ( + Violation( + Path(rel_paths[0]), + 0, + 0, + "LIT000", + f"mypy could not build: {joined}", + ), + ) + idmap = {id(expr): t for expr, t in res.types.items()} + # Resolve trees to absolute source paths while cwd is the build dir, since + # mypy stores the paths it was given (relative to this cwd). + trees: dict[str, Node] = {} + for state in res.graph.values(): + if state.path and state.tree is not None: + trees[os.path.realpath(state.path)] = state.tree + finally: + os.chdir(prev_cwd) + + out: list[Violation] = [] + for rel in rel_paths: + abs_path = (LITELLM_DIR / rel).resolve() + report_path = abs_path.relative_to(REPO_ROOT) + if _is_boundary(report_path): + continue + try: + source = abs_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + out.append( + Violation(report_path, 0, 0, "LIT000", f"could not read file: {exc}") + ) + continue + + ok_lines, ok_violations = scan_any_ok(report_path, source) + out.extend(ok_violations) + tree = trees.get(os.path.realpath(abs_path)) + if tree is None: + continue + for line, col, typ in find_any_in_tree(tree, idmap): + if line in ok_lines: + continue + out.append( + Violation( + report_path, + line, + col, + "LIT009", + f"value type contains Any -> {typ}", + ) + ) + return tuple(out) + + +# --------------------------------------------------------------------------- # +# File selection (changed-only, changed-lines) + driver +# --------------------------------------------------------------------------- # + + +class _AllLines: + """Sentinel: a wholly new / untracked file -- every line is in scope. + + A distinct object, not None, so that `line_map.get(path)` returning None for + a path absent from the map is never mistaken for "whole file in scope".""" + + +# A changed file's in-scope lines: a specific set, or every line. +LineScope = set[int] | _AllLines +ALL_LINES = _AllLines() + + +def _is_boundary(path: Path) -> bool: + posix = path.as_posix() + return any(frag in posix for frag in BOUNDARY_PATHS) + + +def _git(*args: str) -> list[str]: + result = subprocess.run( + ["git", "-C", str(REPO_ROOT), *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.splitlines() + + +def _parse_added_lines(diff_text: str) -> dict[str, set[int]]: + """Map repo-relative path -> set of new-file line numbers the diff adds/edits.""" + changed: dict[str, set[int]] = {} + path: str | None = None + for line in diff_text.splitlines(): + if line.startswith("+++ b/"): + path = line[6:] + elif path and (m := _HUNK_RE.match(line)): + start = int(m.group(1)) + count = int(m.group(2)) if m.group(2) is not None else 1 + if count: + changed.setdefault(path, set()).update(range(start, start + count)) + return changed + + +def changed_line_map(base: str) -> dict[str, LineScope] | None: + """Repo-relative `.py` path under litellm/ -> changed line numbers (or + ALL_LINES for untracked files). Compares the working tree to the merge-base + with `base`, so it covers committed-on-branch + unstaged edits. None if git + is unavailable / not a repo.""" + try: + merge_base = _git("merge-base", base, "HEAD") + point = merge_base[0].strip() if merge_base else base + diff = "\n".join( + _git( + "diff", + "--unified=0", + "--no-color", + "--diff-filter=d", + point, + "--", + "litellm", + ) + ) + untracked = _git("ls-files", "--others", "--exclude-standard", "--", "litellm") + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + out: dict[str, LineScope] = {} + for name, lines in _parse_added_lines(diff).items(): + if name.endswith(".py") and (REPO_ROOT / name).exists(): + out[name] = lines + for name in untracked: + if name.endswith(".py") and (REPO_ROOT / name).exists(): + out[name] = ALL_LINES + return out + + +def _to_litellm_relative(paths: Iterable[Path]) -> list[str]: + rels: list[str] = [] + for p in sorted(paths): + try: + rels.append(p.resolve().relative_to(LITELLM_DIR).as_posix()) + except ValueError: + continue + return rels + + +def _in_scope(v: Violation, line_map: dict[str, LineScope] | None) -> bool: + """A finding survives if line filtering is off (explicit paths), it's a build + error, or its line is one the diff added/edited.""" + if line_map is None or v.code == "LIT000": + return True + lines = line_map.get(v.path.as_posix()) + return lines is ALL_LINES or (lines is not None and v.line in lines) + + +def main(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + description="Any-discipline gate (changed-only, changed-lines)." + ) + parser.add_argument( + "paths", + nargs="*", + help="explicit files (repo-root relative); whole-file, no line filter", + ) + parser.add_argument( + "--changed", + action="store_true", + help="check changed lines under litellm/ vs --base", + ) + parser.add_argument("--base", default=os.environ.get("ANY_GATE_BASE", DEFAULT_BASE)) + args = parser.parse_args(list(argv)) + + line_map: dict[str, LineScope] | None = None + if args.changed: + line_map = changed_line_map(args.base) + if line_map is None: + print( + "check_any_discipline: not a git repository; nothing to check", + file=sys.stderr, + ) + return 0 + rel_paths = _to_litellm_relative( + (REPO_ROOT / name).resolve() for name in line_map + ) + elif args.paths: + rel_paths = _to_litellm_relative((REPO_ROOT / p).resolve() for p in args.paths) + else: + parser.error("pass --changed or explicit file paths") + return 2 + + if not rel_paths: + print("OK: no changed Python lines under litellm/ to check") + return 0 + + violations = tuple(v for v in check_files(rel_paths) if _in_scope(v, line_map)) + + for v in sorted(violations): + print(v.render()) + + if violations: + n = len(violations) + print( + f"\nFAIL: {n} Any-discipline violation(s) on changed lines.\n" + "Give the value a concrete type, or annotate the line `# any-ok: `.", + file=sys.stderr, + ) + return 1 + print( + f"OK: {len(rel_paths)} changed file(s) under litellm/ have no Any-typed values on changed lines" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py new file mode 100644 index 00000000000..5ff485f0b0f --- /dev/null +++ b/scripts/type_check_gate.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Per-rule count gate for mypy and basedpyright. + +Each tool's output is reduced to a count of errors per *rule* (mypy error codes +like ``arg-type``, basedpyright rules like ``reportAny``) and checked against a +committed budget of the form ``{rule: {baseline, slack}}``, the same shape as +``ruff-strict-budget.json``. A rule fails when its codebase-wide total exceeds +``baseline + slack``. Counts ignore file, line, and column, so a violation +moving anywhere in the tree is invisible; only the per-rule total moves the +needle. + +Unlike ``ruff_strict_gate.py`` this does *not* re-run the tool on the merge base +to compute a delta: a second mypy/basedpyright pass is minutes and gigabytes, +whereas ruff is milliseconds. The committed budget is the baseline instead -- +exactly how the previous per-file gate worked -- so keep it fresh with +``--update`` (ratchet), which re-captures every rule's count from the current +tree while preserving each rule's slack. Tool output is read from stdin, so the +caller decides how to invoke the tool (and from which cwd). + +mypy is parsed from its text output (one error per line, the rule code in a +trailing ``[bracket]``). basedpyright is parsed from ``--outputjson``: its text +diagnostics routinely wrap across lines, leaving the ``(reportRule)`` on a +continuation line away from the ``- error:`` marker, so line parsing +mis-attributes ~60% of errors -- the JSON carries an unambiguous ``rule`` field. +""" + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Iterable, Mapping, NamedTuple + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# mypy: one error per line, e.g. `path:12: error: msg [arg-type]`. ERROR_LINE +# recognizes the line; MYPY_CODE pulls the trailing [code]. Kept separate so an +# error emitted without a code is still counted (under UNCODED), never dropped. +MYPY_ERROR = re.compile(r"^(?P.+?):\d+: error:") +MYPY_CODE = re.compile(r"\[(?P[a-z][a-z0-9-]*)\]\s*$") + +# Bucket for an error whose rule code we couldn't read (a mypy error with no +# code, or a basedpyright diagnostic with no `rule`). Counted so it's gated. +UNCODED = "" + +# Ceiling for a rule that shows up at HEAD but isn't in the budget at all -- a +# brand-new error category (new construct, or a tool/version change). baseline +# is treated as 0, so the rule fails once it clears this much slack. +DEFAULT_SLACK = 10 + + +class Breach(NamedTuple): + code: str + total: int + cap: int + + +def _seed_slack(baseline: int) -> int: + """Slack written for a rule first captured into a budget; busy rules get + more headroom, mirroring the tiering in ruff-strict-budget.json. Existing + rules keep whatever slack their JSON already declares.""" + return 10 if baseline >= 50 else 3 + + +def _to_repo_relative(raw: str) -> str | None: + path = Path(raw) + absolute = path if path.is_absolute() else Path.cwd() / path + try: + return absolute.resolve().relative_to(REPO_ROOT).as_posix() + except ValueError: + return None + + +def count_mypy(lines: Iterable[str]) -> dict[str, int]: + """Count in-repo mypy errors per rule code from text output. Errors for + files outside the repo (third-party stubs) are ignored, as before.""" + counts: Counter[str] = Counter() + for raw in lines: + line = raw.rstrip("\n") + match = MYPY_ERROR.match(line) + if match is None or _to_repo_relative(match.group("file")) is None: + continue + code = MYPY_CODE.search(line) + counts[code.group("code") if code else UNCODED] += 1 + return dict(counts) + + +def count_basedpyright(payload: str) -> dict[str, int]: + """Count in-repo basedpyright errors per rule from `--outputjson`. Warnings + and information are ignored; only `severity == "error"` is gated.""" + try: + data = json.loads(payload or "{}") + except json.JSONDecodeError as exc: + sys.stderr.write( + f"basedpyright did not emit valid JSON ({exc}); it likely crashed or " + f"printed text before the JSON. First 500 chars of its output:\n" + f"{payload[:500]}\n" + ) + raise SystemExit(1) from exc + counts: Counter[str] = Counter() + for diag in data.get("generalDiagnostics", []): + if diag.get("severity") != "error": + continue + if _to_repo_relative(diag.get("file", "")) is None: + continue + counts[diag.get("rule") or UNCODED] += 1 + return dict(counts) + + +def count_errors(stdin_text: str, tool: str) -> dict[str, int]: + if tool == "basedpyright": + return count_basedpyright(stdin_text) + return count_mypy(stdin_text.splitlines()) + + +def evaluate( + counts: Mapping[str, int], budget: Mapping[str, Mapping[str, int]] +) -> list[Breach]: + breaches = [] + for code, total in counts.items(): + spec = budget.get(code) + cap = spec["baseline"] + spec["slack"] if spec else DEFAULT_SLACK + if total > cap: + breaches.append(Breach(code, total, cap)) + return sorted(breaches) + + +def is_vacuous_run( + counts: Mapping[str, int], budget: Mapping[str, Mapping[str, int]] +) -> bool: + """True when nothing was parsed but the budget expects errors -- the + signature of a type checker that crashed or produced no output. The CI pipe + swallows the tool's exit code (`tool || true`), so without this guard an + empty run would clear every ceiling and pass silently.""" + return not counts and any(spec["baseline"] for spec in budget.values()) + + +def budget_path(tool: str) -> Path: + return REPO_ROOT / f"{tool}-code-budget.json" + + +def cmd_update(tool: str, counts: Mapping[str, int]) -> None: + path = budget_path(tool) + existing = json.loads(path.read_text()) if path.exists() else {} + budget = { + code: { + "baseline": count, + "slack": ( + existing[code]["slack"] if code in existing else _seed_slack(count) + ), + } + for code, count in sorted(counts.items()) + } + path.write_text(json.dumps(budget, indent=2, sort_keys=True) + "\n") + print( + f"Re-captured {tool} per-rule budget: {len(budget)} rules, {sum(counts.values())} errors total" + ) + + +def cmd_check(tool: str, counts: Mapping[str, int]) -> None: + budget = json.loads(budget_path(tool).read_text()) + if is_vacuous_run(counts, budget): + expected = sum(spec["baseline"] for spec in budget.values()) + print( + f"FAIL: {tool} produced no errors, but {budget_path(tool).name} expects " + f"~{expected}. The type checker almost certainly crashed or emitted " + f"nothing; refusing to certify a vacuous run." + ) + raise SystemExit(1) + breaches = evaluate(counts, budget) + if not breaches: + print( + f"OK: every rule is within its {tool} ceiling ({sum(counts.values())} errors total)" + ) + return + print(f"FAIL: {tool} errors exceed the per-rule ceiling:") + for breach in breaches: + print(f" {breach.code}: {breach.total} errors over cap {breach.cap}") + print( + f"Resolve the new errors, or run 'make lint-{tool}-budget-update' if the ceiling should move." + ) + raise SystemExit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tool", choices=("mypy", "basedpyright"), required=True) + parser.add_argument("--update", action="store_true") + args = parser.parse_args() + counts = count_errors(sys.stdin.read(), args.tool) + cmd_update(args.tool, counts) if args.update else cmd_check(args.tool, counts) + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/test_check_any_discipline.py b/tests/test_litellm/test_check_any_discipline.py new file mode 100644 index 00000000000..d022eebac07 --- /dev/null +++ b/tests/test_litellm/test_check_any_discipline.py @@ -0,0 +1,41 @@ +import importlib.util +from pathlib import Path + +_MODULE_PATH = ( + Path(__file__).resolve().parents[2] / "scripts" / "check_any_discipline.py" +) +_spec = importlib.util.spec_from_file_location("check_any_discipline", _MODULE_PATH) +mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(mod) + +Violation = mod.Violation + + +def _v(path="litellm/x.py", line=10, code="LIT009"): + return Violation(Path(path), line, 0, code, "Any-typed value") + + +def test_violation_on_a_changed_line_is_in_scope(): + assert mod._in_scope(_v(line=10), {"litellm/x.py": {10, 11}}) is True + + +def test_violation_on_an_unchanged_line_of_a_changed_file_is_out_of_scope(): + assert mod._in_scope(_v(line=99), {"litellm/x.py": {10, 11}}) is False + + +def test_whole_new_file_puts_every_line_in_scope(): + assert mod._in_scope(_v(line=99999), {"litellm/x.py": mod.ALL_LINES}) is True + + +def test_file_absent_from_line_map_is_out_of_scope(): + # Regression: ALL_LINES is a distinct sentinel, so a path missing from the map + # (line_map.get -> None) is NOT mistaken for "whole file in scope". + assert mod._in_scope(_v(path="litellm/other.py"), {"litellm/x.py": {1}}) is False + + +def test_no_line_map_means_no_line_filtering(): + assert mod._in_scope(_v(line=12345), None) is True + + +def test_build_error_is_always_in_scope(): + assert mod._in_scope(_v(code="LIT000", line=1), {"litellm/x.py": {2}}) is True diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py new file mode 100644 index 00000000000..eb01bd3b93e --- /dev/null +++ b/tests/test_litellm/test_type_check_gate.py @@ -0,0 +1,133 @@ +import importlib.util +import json +from pathlib import Path + +_MODULE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "type_check_gate.py" +_spec = importlib.util.spec_from_file_location("type_check_gate", _MODULE_PATH) +gate = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(gate) + +ROOT = gate.REPO_ROOT + + +def test_mypy_counts_per_code_ignoring_lines_notes_and_summary(): + text = "\n".join( + [ + f"{ROOT}/litellm/utils.py:10: error: missing annotation [no-untyped-def]", + f"{ROOT}/litellm/utils.py:9999: error: missing annotation [no-untyped-def]", + f"{ROOT}/litellm/main.py:5: error: Returning Any [no-any-return]", + f"{ROOT}/litellm/main.py:5: note: see here", + "Found 3 errors in 2 files (checked 100 source files)", + ] + ) + assert gate.count_errors(text, "mypy") == { + "no-untyped-def": 2, + "no-any-return": 1, + } + + +def _bpr(file, severity, rule): + diag = {"file": str(file), "severity": severity, "message": "msg"} + if rule is not None: + diag["rule"] = rule + return diag + + +def test_basedpyright_counts_per_rule_from_json_not_warnings(): + # basedpyright wraps long messages across lines, so the (reportRule) lands on + # a continuation line away from the `- error:` marker; --outputjson avoids it. + payload = json.dumps( + { + "generalDiagnostics": [ + _bpr(f"{ROOT}/litellm/utils.py", "error", "reportUnknownVariableType"), + _bpr(f"{ROOT}/litellm/utils.py", "error", "reportUnknownVariableType"), + _bpr(f"{ROOT}/litellm/main.py", "error", "reportArgumentType"), + _bpr(f"{ROOT}/litellm/main.py", "warning", "reportUnusedImport"), + ] + } + ) + assert gate.count_errors(payload, "basedpyright") == { + "reportUnknownVariableType": 2, + "reportArgumentType": 1, + } + + +def test_basedpyright_error_without_a_rule_is_bucketed(): + payload = json.dumps( + {"generalDiagnostics": [_bpr(f"{ROOT}/litellm/x.py", "error", None)]} + ) + assert gate.count_errors(payload, "basedpyright") == {gate.UNCODED: 1} + + +def test_mypy_error_without_a_code_is_bucketed_so_it_is_still_gated(): + text = f"{ROOT}/litellm/x.py:1: error: something broke with no code" + assert gate.count_errors(text, "mypy") == {gate.UNCODED: 1} + + +def test_paths_outside_repo_are_skipped(): + text = "/tmp/elsewhere.py:1: error: missing annotation [no-untyped-def]" + assert gate.count_errors(text, "mypy") == {} + payload = json.dumps( + { + "generalDiagnostics": [ + _bpr("/tmp/elsewhere.py", "error", "reportArgumentType") + ] + } + ) + assert gate.count_errors(payload, "basedpyright") == {} + + +def test_at_or_under_ceiling_passes(): + budget = {"no-any-return": {"baseline": 5, "slack": 0}} + assert gate.evaluate({"no-any-return": 5}, budget) == [] + + +def test_one_more_error_than_ceiling_fails(): + budget = {"no-any-return": {"baseline": 5, "slack": 0}} + assert gate.evaluate({"no-any-return": 6}, budget) == [ + gate.Breach("no-any-return", 6, 5) + ] + + +def test_slack_absorbs_small_increase_then_fails_past_it(): + budget = {"arg-type": {"baseline": 5, "slack": 5}} + assert gate.evaluate({"arg-type": 10}, budget) == [] + assert gate.evaluate({"arg-type": 11}, budget) == [gate.Breach("arg-type", 11, 10)] + + +def test_unbudgeted_new_code_uses_default_slack(): + assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK}, {}) == [] + assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK + 1}, {}) == [ + gate.Breach("brand-new", gate.DEFAULT_SLACK + 1, gate.DEFAULT_SLACK) + ] + + +def test_no_output_against_a_nonempty_budget_is_a_vacuous_run(): + # A crashed type checker emits nothing; the gate must not certify it as clean. + budget = {"no-untyped-def": {"baseline": 4888, "slack": 10}} + assert gate.is_vacuous_run({}, budget) is True + + +def test_genuine_zero_and_empty_budget_are_not_vacuous(): + assert gate.is_vacuous_run({}, {}) is False + assert ( + gate.is_vacuous_run({}, {"no-untyped-def": {"baseline": 0, "slack": 3}}) + is False + ) + assert ( + gate.is_vacuous_run({"arg-type": 1}, {"arg-type": {"baseline": 9, "slack": 1}}) + is False + ) + + +def test_malformed_basedpyright_json_exits_loudly_not_as_zero_errors(): + import pytest + + with pytest.raises(SystemExit): + gate.count_errors("startup warning\n{not json", "basedpyright") + + +def test_empty_basedpyright_payload_counts_zero(): + # Empty (not malformed) output parses to zero; the vacuous-run guard, not the + # parser, is what rejects an empty run. + assert gate.count_errors("", "basedpyright") == {} diff --git a/uv.lock b/uv.lock index 0efaa74cddb..bc796e6ed07 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-10T00:35:00.40525Z" +exclude-newer = "2026-06-11T06:56:06.940919973Z" exclude-newer-span = "P3D" [manifest] @@ -540,6 +540,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, ] +[[package]] +name = "basedpyright" +version = "1.39.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodejs-wheel-binaries" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/e5/0d685b5808436c628ab8b9edad6810b889d11044a962bc42b128543910ea/basedpyright-1.39.7.tar.gz", hash = "sha256:688d913a19c417870c164c630ed9cdd83a8d8b484b30ab8e99f5dec4ae9604a6", size = 25503256, upload-time = "2026-06-07T11:33:27.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/f4/5b1e8ea279ce8f97a6bb1518c84fa25f5794022053ce10eab22ad3f0b51b/basedpyright-1.39.7-py3-none-any.whl", hash = "sha256:81266deb6044c9be98fb4555e4b7b1a521d8aee06b66e80858d183b0e3991140", size = 13182666, upload-time = "2026-06-07T11:33:24.119Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -3416,13 +3428,13 @@ ci = [ { name = "pyarrow" }, { name = "pygithub" }, { name = "pylint" }, - { name = "pyright" }, { name = "pytest-codspeed" }, { name = "pytest-retry" }, { name = "tenacity" }, { name = "traceloop-sdk" }, ] dev = [ + { name = "basedpyright" }, { name = "black" }, { name = "diff-cover" }, { name = "fakeredis" }, @@ -3584,13 +3596,13 @@ ci = [ { name = "pyarrow", specifier = "==23.0.1" }, { name = "pygithub", specifier = "==2.8.1" }, { name = "pylint", specifier = "==4.0.5" }, - { name = "pyright", specifier = "==1.1.408" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, { name = "pytest-retry", specifier = "==1.7.0" }, { name = "tenacity", specifier = "==8.5.0" }, { name = "traceloop-sdk", specifier = "==0.33.12" }, ] dev = [ + { name = "basedpyright", specifier = "==1.39.7" }, { name = "black", specifier = "==26.3.1" }, { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, @@ -4270,6 +4282,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "nodejs-wheel-binaries" +version = "24.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/22/2a5beb4e21417c73233d9f65cf6f3e96e891b80d2f550a8f630ebc6b88c6/nodejs_wheel_binaries-24.16.0.tar.gz", hash = "sha256:c973cb69dc5fd16e6f6dc6e579e2c3d5534e2a1f57619dddf5ba070efa7dde37", size = 8056, upload-time = "2026-05-30T16:52:09.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/d1/68b43b53cd0fa83ae6fd406705023ca988d9e0ca41c724d82e66fbeb2ef6/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_arm64.whl", hash = "sha256:d9f8f677dcf30e37ac244f07869726abe043f01eb0f45722b1df31cc2af7093c", size = 55666374, upload-time = "2026-05-30T16:51:39.588Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b2/40a989159599080da485de966c4c2d207e852ac7aa7864702626d96c8bf5/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_x86_64.whl", hash = "sha256:3d0370fe7120ce9697a4f60d40480d2bd8808d9f30131458d5afc0040d4e5a51", size = 55838487, upload-time = "2026-05-30T16:51:43.383Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a7/cd42174fb5ff6faff7fa8d326a18914d8f232098ab5de055b57c16fa13ca/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:85dc92bbb79c851569c5925dcc2a4c915a034efab375f99e4e7e6bbe9cca8342", size = 60179540, upload-time = "2026-05-30T16:51:47.036Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/c8a1f9ae140aa28df8744d984d01d4b3af7cdd6555af12127f40ceb45a7d/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:2f3036292811514ba847b3708492644764f88a833ac425c5f55007014308ddfd", size = 60716262, upload-time = "2026-05-30T16:51:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/64/c9/7c35b3737f59e36d0249c265397b7bff570519b95301d6e16ea361e904ad/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:db8a8a76ebd2b28ecbfc9ad464baa3707241b9e050a30e2efdf6f60c0f886502", size = 62230592, upload-time = "2026-05-30T16:51:55Z" }, + { url = "https://files.pythonhosted.org/packages/04/96/d931255cf9d11a84d6b54d882dba7434646467d568ccf070ea3418638df3/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f1a3d8f7b4491cbbd023ba3fc4e901fcca2d9fb80d57f24ba3890de8b1dbac03", size = 62841759, upload-time = "2026-05-30T16:51:59.407Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7b/8b7a3f41bc255411be30b6d7d288aab8ffd9ea2055db8555ced3548007b9/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_amd64.whl", hash = "sha256:bb136be9944f0662dcf1120f45193a6b75b13fac378971a95cc42c9f879a81aa", size = 42027734, upload-time = "2026-05-30T16:52:03.348Z" }, + { url = "https://files.pythonhosted.org/packages/17/66/1ed71f1f529b8ca727d42c7ceb9db0bef145ce4a13dfc86fb50aa44f3be6/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_arm64.whl", hash = "sha256:8308940b5edd0a50dc5267ea36ba21c9f668e83fe0d9f293937174d3a7e31c36", size = 39714528, upload-time = "2026-05-30T16:52:06.421Z" }, +] + [[package]] name = "numpy" version = "1.26.4" @@ -6082,19 +6110,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, ] -[[package]] -name = "pyright" -version = "1.1.408" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nodeenv" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/b2/5db700e52554b8f025faa9c3c624c59f1f6c8841ba81ab97641b54322f16/pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684", size = 4400578, upload-time = "2026-01-08T08:07:38.795Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/82/a2c93e32800940d9573fb28c346772a14778b84ba7524e691b324620ab89/pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1", size = 6399144, upload-time = "2026-01-08T08:07:37.082Z" }, -] - [[package]] name = "pyroscope-io" version = "0.8.16"