test(ci): prove RULE_CODES by running every checker rule

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

scripts/budget_ratchet_check.py

@@ -156,7 +156,7 @@ def _load_script(name: str) -> ModuleType:
     return module
 
 
-def retired_rules(rel: str, base: dict) -> frozenset[str]:
+def retired_rules(rel: str, base: dict[str, object]) -> frozenset[str]:
     """Rules in the base budget that the paired checker can no longer emit, so there is no ceiling to loosen."""
     source: Final = RETIREMENT_SOURCES.get(rel)
     if source is None:

tests/test_litellm/test_check_test_quality.py

@@ -7,13 +7,13 @@ produced against tests/e2e, where the assertions live in a shared helper rather
 in the test body.
 """
 
-import ast
 import importlib.util
 import os
-import re
 import subprocess
 import sys
 from pathlib import Path
+from types import MappingProxyType
+from typing import Final
 
 import pytest
 
@@ -614,17 +614,42 @@ def test_a_fanned_out_run_reports_each_generated_file_exactly_once(tmp_path):
     assert all(" TQ001 " in line for line in reported)
 
 
-def test_rule_codes_match_every_code_the_checker_emits():
-    source = _MODULE_PATH.read_text(encoding="utf-8")
-    tree = ast.parse(source)
-    definition = next(
-        node
-        for node in tree.body
-        if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.target.id == "RULE_CODES"
+_VIOLATING_SNIPPETS: Final = MappingProxyType(
+    {
+        "TQ000": ("test_snippet.py", "def test_broken(:\n    pass\n"),
+        "TQ001": ("test_snippet.py", "def test_nothing():\n    compute()\n"),
+        "TQ002": (
+            "test_snippet.py",
+            "from unittest.mock import patch\n"
+            "\n"
+            "\n"
+            "def test_echo():\n"
+            "    with patch('litellm.completion') as mock_completion:\n"
+            "        run()\n"
+            "    mock_completion.assert_called_once()\n",
+        ),
+        "TQ003": ("test_snippet.py", "import sys\n\nsys.path.insert(0, '..')\n"),
+        "TQ004": ("test_snippet.py", "import os\n\nos.environ['KEY'] = 'v'\n"),
+        "TQ005": ("test_snippet.py", "import litellm\n\nlitellm.drop_params = True\n"),
+        "TQ006": ("test_snippet.py", _DIRECT_GATE),
+        "TQ007": ("conftest.py", _SNAPSHOT_CONFTEST),
+        "TQ009": (
+            "test_snippet.py",
+            'import subprocess, sys\nsubprocess.run([sys.executable, "-c", "pass"])\n',
+        ),
+    }
+)
+
+
+def test_rule_codes_match_every_code_the_checker_emits(tmp_path):
+    emitted = frozenset(
+        v.code
+        for name, source in _VIOLATING_SNIPPETS.values()
+        for v in checker.check_file(_written(tmp_path, source, name))
     )
-    lines = source.splitlines()
-    outside = "\n".join(lines[: definition.lineno - 1] + lines[definition.end_lineno :])
-    assert frozenset(re.findall(r'"(TQ\d{3})"', outside)) == checker.RULE_CODES
+    for code, (name, source) in _VIOLATING_SNIPPETS.items():
+        assert code in [v.code for v in checker.check_file(_written(tmp_path, source, name))], code
+    assert emitted == checker.RULE_CODES
 
 
 def test_sys_executable_child_without_isolation_flag_is_flagged(tmp_path):