fix(ci): skip the generated dashboard bundle in the master key guard (#44890)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/test-code-quality.yml

@@ -168,6 +168,9 @@ jobs:
       - name: check_no_publicly_known_master_key
         run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
 
+      - name: test_check_no_publicly_known_master_key
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
+
       - name: check_unbounded_in_lists (fails on findings not in the baseline)
         run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
 

tests/code_coverage_tests/check_no_publicly_known_master_key.py

@@ -9,10 +9,11 @@ from typing import Final
 
 REPO_ROOT: Final = Path(__file__).resolve().parents[2]
 PUBLICLY_KNOWN_MASTER_KEY_PREFIX: Final = "sk-" + "1234"
+GENERATED_DASHBOARD_BUNDLE_PATHSPEC: Final = ":(exclude)litellm/proxy/_experimental/out"
 
 
-def _file_violations(relative_path: str) -> tuple[str, ...]:
-    path: Final = REPO_ROOT / relative_path
+def _file_violations(repo_root: Path, relative_path: str) -> tuple[str, ...]:
+    path: Final = repo_root / relative_path
     if path.is_dir():
         return ()
     try:
@@ -32,25 +33,26 @@ def _file_violations(relative_path: str) -> tuple[str, ...]:
     )
 
 
-def _violations(tracked_paths: tuple[str, ...]) -> Iterator[str]:
-    for path in tracked_paths:
-        yield from _file_violations(path)
-
-
-def main() -> int:
+def _tracked_paths(repo_root: Path) -> tuple[str, ...]:
     result: Final = subprocess.run(
-        ["git", "-C", os.fspath(REPO_ROOT), "ls-files", "-z"],
+        ["git", "-C", os.fspath(repo_root), "ls-files", "-z", "--", ".", GENERATED_DASHBOARD_BUNDLE_PATHSPEC],
         check=True,
         stdout=subprocess.PIPE,
     )
-    tracked_paths: Final = tuple(
-        os.fsdecode(path) for path in result.stdout.split(b"\0") if path
-    )
-    violations: Final = tuple(_violations(tracked_paths))
-    for violation in violations:
+    return tuple(os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
+
+
+def violations(repo_root: Path) -> Iterator[str]:
+    for path in _tracked_paths(repo_root):
+        yield from _file_violations(repo_root, path)
+
+
+def main() -> int:
+    violations_found: Final = tuple(violations(REPO_ROOT))
+    for violation in violations_found:
         print(violation)
-    if violations:
-        print(f"\n{len(violations)} tracked line(s) contain the publicly known master key prefix")
+    if violations_found:
+        print(f"\n{len(violations_found)} tracked line(s) contain the publicly known master key prefix")
         return 1
     print("No tracked files contain the publicly known master key prefix")
     return 0

tests/code_coverage_tests/test_check_no_publicly_known_master_key.py

@@ -0,0 +1,58 @@
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+GUARD: Final = Path(__file__).resolve().parent / "check_no_publicly_known_master_key.py"
+RETIRED_KEY: Final = "sk-" + "1234"
+BUNDLE_CHUNK: Final = "litellm/proxy/_experimental/out/_next/static/chunks/chunk.js"
+
+
+def _git(repo: Path, *args: str) -> None:
+    subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True)
+
+
+def _repo_with(tmp_path: Path, files: dict[str, str]) -> Path:
+    guard_copy: Final = tmp_path / "tests" / "code_coverage_tests" / GUARD.name
+    guard_copy.parent.mkdir(parents=True)
+    shutil.copy(GUARD, guard_copy)
+    for relative_path, contents in files.items():
+        target = tmp_path / relative_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(contents)
+    _git(tmp_path, "init", "-q")
+    _git(tmp_path, "add", "-A")
+    return guard_copy
+
+
+def _run(guard: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run([sys.executable, str(guard)], capture_output=True, text=True)
+
+
+def test_generated_dashboard_bundle_is_not_flagged(tmp_path: Path) -> None:
+    guard: Final = _repo_with(tmp_path, {BUNDLE_CHUNK: f'api_key="{RETIRED_KEY}"\n', "docs/a.md": "clean\n"})
+
+    result: Final = _run(guard)
+
+    assert result.returncode == 0, result.stdout
+    assert BUNDLE_CHUNK not in result.stdout
+
+
+@pytest.mark.parametrize(
+    "source_path",
+    ["litellm/proxy/proxy_server.py", "ui/litellm-dashboard/src/app/page.tsx", "litellm/proxy/_experimental/x.py"],
+)
+def test_source_files_are_still_flagged(tmp_path: Path, source_path: str) -> None:
+    guard: Final = _repo_with(
+        tmp_path,
+        {BUNDLE_CHUNK: f'api_key="{RETIRED_KEY}"\n', source_path: f'key = "{RETIRED_KEY}"\n'},
+    )
+
+    result: Final = _run(guard)
+
+    assert result.returncode == 1
+    assert f"{source_path}:1" in result.stdout
+    assert BUNDLE_CHUNK not in result.stdout