build(make): run CI's code-quality scripts in make lint

Makefile

@@ -239,6 +239,9 @@ lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 check-circular-imports: $(LINT_DEP_INSTALL)
 	cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
+lint-code-quality: $(LINT_DEP_INSTALL)
+	$(UV_RUN) python scripts/code_quality_checks.py
+
 check-import-safety: $(LINT_DEP_INSTALL)
 	@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
@@ -246,7 +249,7 @@ check-import-safety: $(LINT_DEP_INSTALL)
 # green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then
 # runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
 # type-discipline / basedpyright gates as a delta vs the base, then the circular-import
-# and import-safety checks. Steps that compare against the base resolve it the same way CI
+# and import-safety checks, plus the tests/code_coverage_tests scripts of its code-quality job. Steps that compare against the base resolve it the same way CI
 # does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
 # base fetch) runs once up front; the checks themselves are independent, so a sub-make
 # fans them out with -j and the fast ones finish under basedpyright's shadow.
@@ -257,7 +260,7 @@ lint-inner: lint-install
 	@base_ref=$$($(RESOLVE_BASE)) && \
 	$(MAKE) BASE_REF="$$base_ref" -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
 
-lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-openapi-docs lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
+lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-openapi-docs lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety lint-code-quality
 
 # Faster linting for local development (only checks changed code)
 lint-dev: lint-format-changed check-circular-imports check-import-safety

scripts/code_quality_checks.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Run the tests/code_coverage_tests scripts that test-linting.yml's code-quality job runs.
+
+The list is read from the workflow itself, so a script added to that job runs under
+`make lint` (and so `make check`) as soon as it lands, and a local pass means those
+CI steps pass too.
+"""
+
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import Final
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+REPO_ROOT: Final = Path(__file__).resolve().parent.parent
+WORKFLOW: Final = REPO_ROOT / ".github" / "workflows" / "test-linting.yml"
+JOB: Final = "code-quality"
+SCRIPT_STEP: Final = re.compile(r"uv run --no-sync python (?P<script>\./tests/code_coverage_tests/[\w/]+\.py)")
+REWRITES_TRACKED_FILES: Final = frozenset({"./tests/code_coverage_tests/check_licenses.py"})
+
+
+class Step(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    run: str = ""
+
+
+class Job(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    steps: tuple[Step, ...] = ()
+
+
+class Workflow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    jobs: Mapping[str, Job] = Field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class Outcome:
+    script: str
+    returncode: int
+    output: str
+
+
+def scripts(workflow_source: str) -> tuple[str, ...]:
+    job: Final = Workflow.model_validate(yaml.safe_load(workflow_source)).jobs.get(JOB, Job())
+    matches: Final = tuple(SCRIPT_STEP.fullmatch(step.run.strip()) for step in job.steps)
+    return tuple(match["script"] for match in matches if match and match["script"] not in REWRITES_TRACKED_FILES)
+
+
+def run(script: str, cwd: Path) -> Outcome:
+    completed: Final = subprocess.run((sys.executable, script), cwd=cwd, capture_output=True, text=True, check=False)
+    return Outcome(script, completed.returncode, completed.stdout + completed.stderr)
+
+
+def failures(selected: Sequence[str], cwd: Path) -> tuple[Outcome, ...]:
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as pool:
+        outcomes: Final = tuple(pool.map(partial(run, cwd=cwd), selected))
+    return tuple(outcome for outcome in outcomes if outcome.returncode != 0)
+
+
+def main(workflow: Path = WORKFLOW, cwd: Path = REPO_ROOT) -> int:
+    selected: Final = scripts(workflow.read_text())
+    if not selected:
+        sys.stderr.write(f"✗ No tests/code_coverage_tests scripts found in the {JOB} job of {workflow}\n")
+        return 1
+    failed: Final = failures(selected, cwd)
+    sys.stderr.write(
+        "".join(f"✗ {outcome.script} exited {outcome.returncode}\n{outcome.output}\n" for outcome in failed)
+    )
+    sys.stdout.write(f"{JOB}: {len(selected) - len(failed)} of {len(selected)} scripts passed\n")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/unit/test_code_quality_checks.py

@@ -0,0 +1,75 @@
+import importlib.util
+from pathlib import Path
+
+_MODULE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "code_quality_checks.py"
+_spec = importlib.util.spec_from_file_location("code_quality_checks", _MODULE_PATH)
+mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(mod)
+
+WORKFLOW_SOURCE = """
+jobs:
+  python:
+    steps:
+      - run: uv run --no-sync python ./tests/code_coverage_tests/from_another_job.py
+  code-quality:
+    steps:
+      - uses: actions/checkout@v4
+      - name: recursive_detector
+        run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
+      - run: uv run --no-sync pytest -q tests/code_coverage_tests/test_thing.py
+      - run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+      - run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
+      - run: |
+          uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
+"""
+
+
+def test_selects_the_code_quality_jobs_scripts_that_leave_the_tree_untouched():
+    assert mod.scripts(WORKFLOW_SOURCE) == (
+        "./tests/code_coverage_tests/recursive_detector.py",
+        "./tests/code_coverage_tests/router_code_coverage.py",
+    )
+
+
+def test_every_script_the_real_code_quality_job_names_exists():
+    selected = mod.scripts(mod.WORKFLOW.read_text())
+    assert selected
+    assert [script for script in selected if not (mod.REPO_ROOT / script).is_file()] == []
+
+
+def _workspace(tmp_path: Path, bad_exit: int) -> Path:
+    scripts_dir = tmp_path / "tests" / "code_coverage_tests"
+    scripts_dir.mkdir(parents=True)
+    (scripts_dir / "passes.py").write_text("print('fine')\n")
+    (scripts_dir / "flags.py").write_text(f"import sys\nprint('helper calls itself')\nsys.exit({bad_exit})\n")
+    workflow = tmp_path / "test-linting.yml"
+    workflow.write_text(
+        "jobs:\n  code-quality:\n    steps:\n"
+        "      - run: uv run --no-sync python ./tests/code_coverage_tests/passes.py\n"
+        "      - run: uv run --no-sync python ./tests/code_coverage_tests/flags.py\n"
+    )
+    return workflow
+
+
+def test_fails_and_shows_the_output_of_a_failing_script(tmp_path, capsys):
+    workflow = _workspace(tmp_path, bad_exit=3)
+
+    assert mod.main(workflow, tmp_path) == 1
+    captured = capsys.readouterr()
+    assert "./tests/code_coverage_tests/flags.py exited 3" in captured.err
+    assert "helper calls itself" in captured.err
+    assert "passes.py" not in captured.err
+    assert "1 of 2 scripts passed" in captured.out
+
+
+def test_passes_when_every_script_passes(tmp_path):
+    workflow = _workspace(tmp_path, bad_exit=0)
+
+    assert mod.main(workflow, tmp_path) == 0
+
+
+def test_fails_when_the_job_names_no_scripts(tmp_path):
+    workflow = tmp_path / "test-linting.yml"
+    workflow.write_text("jobs:\n  python:\n    steps:\n      - run: make lint\n")
+
+    assert mod.main(workflow, tmp_path) == 1