From 65df36c4c3b26b664d9bd2004b657ad214d19b59 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 20 Aug 2026 11:28:27 -0700 Subject: [PATCH] feat(ci): scope mutation testing to the pull request diff The existing mutation workflow is manual and mutates a whole folder, which takes hours, so nothing about a pull request's own test quality gets checked before it merges. This adds the diff-scoped half, following Google's approach in "State of Mutation Testing at Google": mutate what the change touched, not the codebase. scripts/mutation_diff_scope.py reads the diff against the merge base, rewrites [tool.mutmut] so paths_to_mutate is the changed production files and tests_dir is the tests mirroring them, then emits mutant-name globs for the functions holding the changed lines. mutmut trampolines per function and per method, so that is the smallest unit it can be asked to run. The new workflow is advisory: results land in the job summary and an artifact, and nothing blocks a merge. A --max-functions cap bounds the worst case and prints what it dropped rather than truncating quietly. The report script now counts only the mutants a run actually executed, since a diff-scoped run leaves everything else at "not checked" and the old summary folded those into the score. --- .github/workflows/mutation-test-pr.yml | 158 +++++++++ .gitignore | 9 + scripts/mutation_diff_scope.py | 303 ++++++++++++++++++ scripts/mutation_report.py | 65 +++- .../test_litellm/test_mutation_diff_scope.py | 242 ++++++++++++++ 5 files changed, 762 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/mutation-test-pr.yml create mode 100644 scripts/mutation_diff_scope.py create mode 100644 tests/test_litellm/test_mutation_diff_scope.py diff --git a/.github/workflows/mutation-test-pr.yml b/.github/workflows/mutation-test-pr.yml new file mode 100644 index 00000000000..b7ae605ccd6 --- /dev/null +++ b/.github/workflows/mutation-test-pr.yml @@ -0,0 +1,158 @@ +name: "Mutation Test (PR diff)" + +# Diff-scoped mutation testing, modeled on Google's "State of Mutation Testing +# at Google" (https://research.google/pubs/state-of-mutation-testing-at-google/): +# mutate only what the pull request changed, not the whole codebase. +# +# scripts/mutation_diff_scope.py rewrites [tool.mutmut] so paths_to_mutate is +# the changed production files and tests_dir is the tests mirroring them, then +# emits mutant-name globs for the functions containing the changed lines. +# +# Advisory only: the result is written to the job summary and uploaded as an +# artifact. Nothing here blocks a merge. + +on: + pull_request: + paths: + - "litellm/**.py" + - "scripts/mutation_diff_scope.py" + - ".github/workflows/mutation-test-pr.yml" + +permissions: + contents: read + +concurrency: + group: mutation-test-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + mutation: + name: Mutate the diff + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + # The scope script diffs HEAD against the merge base with the target + # branch, so the full history has to be present. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Scope the run to the diff + id: scope + run: | + python scripts/mutation_diff_scope.py \ + --base "origin/${{ github.event.pull_request.base.ref }}" \ + --write-pyproject | tee mutmut-scope.log + + - name: Set up uv + if: steps.scope.outputs.has_scope == 'true' + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache uv dependencies + if: steps.scope.outputs.has_scope == 'true' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + if: steps.scope.outputs.has_scope == 'true' + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + + - name: Cache Prisma binaries + if: steps.scope.outputs.has_scope == 'true' + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + if: steps.scope.outputs.has_scope == 'true' + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + # See .github/workflows/mutation-test.yml for why these two steps exist: + # an editable install shadows the mutants/ sandbox, and pytest-retry + # crashes under mutmut's in-process pytest.main() call. + - name: Reinstall litellm non-editable (so mutants/ is not shadowed) + if: steps.scope.outputs.has_scope == 'true' + run: | + uv pip uninstall litellm + uv pip install . --no-deps + + - name: Remove pytest plugins that conflict with mutmut + if: steps.scope.outputs.has_scope == 'true' + run: | + uv pip uninstall pytest-retry || true + + - name: Run mutmut on the changed functions + if: steps.scope.outputs.has_scope == 'true' + continue-on-error: true + timeout-minutes: 10 + env: + PYTHONPATH: ${{ github.workspace }}/mutants + run: | + set -o pipefail + # mutants/ has to exist before mutmut starts: it is on PYTHONPATH, and Python + # caches a missing sys.path entry, so a directory created later is never searched. + mkdir -p mutants + GLOBS=() + while IFS= read -r glob || [ -n "$glob" ]; do + [ -n "$glob" ] && GLOBS+=("$glob") + done < mutmut-scope-globs.txt + uv run --no-sync --with mutmut==3.5.0 mutmut run "${GLOBS[@]}" 2>&1 | tee mutmut-run.log + + - name: Generate mutation report + if: always() && steps.scope.outputs.has_scope == 'true' + run: | + set +e + uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1 + uv run --no-sync --with mutmut==3.5.0 mutmut results --all=true > mutmut-results.txt 2>&1 + uv run --no-sync python scripts/mutation_report.py + { + echo "## Mutation testing on this diff" + echo "" + echo '```' + cat mutmut-scope.log + echo '```' + echo "" + head -c 900000 mutation-report.md + } >> "$GITHUB_STEP_SUMMARY" + + - name: Report that nothing was in scope + if: steps.scope.outputs.has_scope != 'true' + run: | + { + echo "## Mutation testing on this diff" + echo "" + echo "No mutable production code changed, or no mirrored tests were found for it." + echo "" + echo '```' + cat mutmut-scope.log + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload mutation artifacts + if: always() && steps.scope.outputs.has_scope == 'true' + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: mutmut-pr-${{ github.run_id }}-${{ github.run_attempt }} + path: | + mutation-report.md + mutmut-results.txt + mutmut-run.log + mutmut-scope.json + mutmut-scope.log + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 201e02f2189..cb1b21a9aef 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,12 @@ crash.*.log ui/litellm-dashboard/out/ litellm.log + +# mutation testing scratch (mutmut sandbox + diff-scope artifacts) +mutants/ +mutation-report.md +mutmut-results.txt +mutmut-run.log +mutmut-scope.json +mutmut-scope-globs.txt +mutmut-scope.log diff --git a/scripts/mutation_diff_scope.py b/scripts/mutation_diff_scope.py new file mode 100644 index 00000000000..4ccc77fceab --- /dev/null +++ b/scripts/mutation_diff_scope.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Scope a mutmut run to the production code a pull request actually changed. + +mutmut reads its configuration only from ``[tool.mutmut]`` in ``pyproject.toml``, +and it filters which mutants to *execute* by fnmatch against mutant names. This +script bridges a git diff to both: + +1. It rewrites ``[tool.mutmut]`` so ``paths_to_mutate`` is the changed production + files and ``tests_dir`` is the test files that mirror them. +2. It emits mutant-name globs for the functions containing the changed lines, to + be passed as arguments to ``mutmut run``. + +mutmut installs its trampolines per top-level function and per class method, so a +changed line's enclosing function is the smallest unit it can execute. + +Usage: + python scripts/mutation_diff_scope.py --base origin/litellm_internal_staging +""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import subprocess +import sys +import tomllib +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent +MUTATE_ROOTS: Final = ("litellm/",) +EXCLUDED_PREFIXES: Final = ( + "litellm/proxy/_experimental/", + "litellm/types/", +) +TEST_MIRROR_ROOT: Final = "tests/test_litellm" +# mutmut runs the selected tests from a `mutants/` sandbox. It copies tests/ itself, but a +# test that reaches back out to repo tooling (scripts/, .github/) fails to import there. +SANDBOX_COPIES: Final = ("litellm/", "scripts/", ".github/") +CLASS_NAME_SEPARATOR: Final = "ǁ" +HUNK_HEADER: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@") +CONFIG_SECTION: Final = re.compile(r"^\[tool\.mutmut\]$.*?(?=^\[)", re.MULTILINE | re.DOTALL) + + +@dataclass(frozen=True, slots=True) +class ChangedFile: + path: str + lines: frozenset[int] + + +@dataclass(frozen=True, slots=True) +class TrampolineUnit: + """A function or method mutmut can address by name, and the lines it spans.""" + + mangled_name: str + start_line: int + end_line: int + + +@dataclass(frozen=True, slots=True) +class Scope: + files: tuple[ChangedFile, ...] + tests: tuple[str, ...] + globs: tuple[str, ...] + dropped_globs: tuple[str, ...] + + @property + def is_runnable(self) -> bool: + return bool(self.globs and self.tests) + + +def _git(root: Path, *args: str) -> str: + return subprocess.run(("git", *args), cwd=root, capture_output=True, text=True, check=True).stdout + + +def is_mutable_path(root: Path, path: str) -> bool: + return ( + path.endswith(".py") + and path.startswith(MUTATE_ROOTS) + and not path.startswith(EXCLUDED_PREFIXES) + and (root / path).is_file() + ) + + +def is_unit_test_path(root: Path, path: str) -> bool: + """Only the mirrored unit suite; tests/e2e needs a live proxy and cannot run in mutmut's sandbox.""" + return ( + path.startswith(f"{TEST_MIRROR_ROOT}/") + and path.endswith(".py") + and Path(path).name.startswith("test_") + and (root / path).is_file() + ) + + +def parse_unified_diff(root: Path, diff: str) -> tuple[ChangedFile, ...]: + """Map each changed production file to the post-image line numbers the diff touched.""" + + def touched_lines() -> Iterator[tuple[str, int]]: + current = "" + cursor = 0 + for line in diff.splitlines(): + if line.startswith("+++ b/"): + current = line[len("+++ b/") :] + elif (header := HUNK_HEADER.match(line)) is not None: + cursor = int(header.group(1)) + elif line.startswith("+") and not line.startswith("+++") and current: + yield current, cursor + cursor += 1 + + touched: Final = tuple(touched_lines()) + return tuple( + ChangedFile(path=path, lines=frozenset(line for other, line in touched if other == path)) + for path in sorted({path for path, _ in touched}) + if is_mutable_path(root, path) + ) + + +def changed_test_files(root: Path, diff: str) -> tuple[str, ...]: + return tuple( + sorted( + path + for line in diff.splitlines() + if line.startswith("+++ b/") and is_unit_test_path(root, path := line[len("+++ b/") :]) + ) + ) + + +def mapped_tests(root: Path, path: str) -> tuple[str, ...]: + """Find the tests mirroring a production file, per tests/test_litellm/readme.md.""" + relative: Final = Path(path).relative_to("litellm") + mirror_dir: Final = Path(TEST_MIRROR_ROOT) / relative.parent + named: Final = tuple(sorted(str(p.relative_to(root)) for p in (root / mirror_dir).glob(f"test_{relative.stem}*.py"))) + if named: + return named + return (str(mirror_dir),) if (root / mirror_dir).is_dir() else () + + +def trampoline_units(source: str) -> tuple[TrampolineUnit, ...]: + """Every unit mutmut trampolines: top-level functions and direct class methods.""" + tree: Final = ast.parse(source) + + def unit(node: ast.stmt, class_name: str | None) -> TrampolineUnit | None: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return None + separator = CLASS_NAME_SEPARATOR + mangled = f"x{separator}{class_name}{separator}{node.name}" if class_name else f"x_{node.name}" + return TrampolineUnit( + mangled_name=mangled, + start_line=min((d.lineno for d in node.decorator_list), default=node.lineno), + end_line=node.end_lineno or node.lineno, + ) + + def walk() -> Iterator[TrampolineUnit]: + for statement in tree.body: + if (function := unit(statement, None)) is not None: + yield function + elif isinstance(statement, ast.ClassDef): + for member in statement.body: + if (method := unit(member, statement.name)) is not None: + yield method + + return tuple(walk()) + + +def module_name(path: str) -> str: + return path[: -len(".py")].replace(os.sep, ".").replace("/", ".") + + +def globs_for(root: Path, changed: ChangedFile) -> tuple[str, ...]: + try: + units = trampoline_units((root / changed.path).read_text(encoding="utf-8")) + except SyntaxError: + return () + module: Final = module_name(changed.path) + return tuple( + f"{module}.{unit.mangled_name}__mutmut_*" + for unit in units + if any(unit.start_line <= line <= unit.end_line for line in changed.lines) + ) + + +def build_scope(root: Path, base: str, max_functions: int) -> Scope: + merge_base: Final = _git(root, "merge-base", base, "HEAD").strip() + diff: Final = _git(root, "diff", "-U0", merge_base, "--", ".") + files: Final = parse_unified_diff(root, diff) + all_globs: Final = tuple(glob for changed in files for glob in globs_for(root, changed)) + tests: Final = tuple( + sorted({*(t for changed in files for t in mapped_tests(root, changed.path)), *changed_test_files(root, diff)}) + ) + return Scope( + files=files, + tests=tests, + globs=all_globs[:max_functions], + dropped_globs=all_globs[max_functions:], + ) + + +def render_config(scope: Scope, pytest_add_cli_args: Sequence[str]) -> str: + def toml_list(values: Iterable[str]) -> str: + return "[\n" + "".join(f' "{value}",\n' for value in values) + "]" + + return ( + "[tool.mutmut]\n" + f"paths_to_mutate = {toml_list(f.path for f in scope.files)}\n" + f"tests_dir = {toml_list(scope.tests)}\n" + f"also_copy = {toml_list(SANDBOX_COPIES)}\n" + # Gathering coverage costs an extra full run of the selected tests, and mutants on + # uncovered lines are cheap here: mutmut marks them "no tests" without running any. + "mutate_only_covered_lines = false\n" + f"pytest_add_cli_args = {toml_list(pytest_add_cli_args)}\n" + "\n" + ) + + +def rewrite_pyproject(root: Path, scope: Scope, pytest_add_cli_args: Sequence[str]) -> None: + pyproject: Final = root / "pyproject.toml" + original: Final = pyproject.read_text(encoding="utf-8") + if CONFIG_SECTION.search(original) is None: + raise SystemExit("Could not find a [tool.mutmut] section to replace in pyproject.toml") + pyproject.write_text( + CONFIG_SECTION.sub(lambda _: render_config(scope, pytest_add_cli_args), original, count=1), encoding="utf-8" + ) + + +def existing_pytest_cli_args(root: Path) -> tuple[str, ...]: + with open(root / "pyproject.toml", "rb") as handle: + section: Mapping[str, object] = tomllib.load(handle)["tool"]["mutmut"] + args: Final = section.get("pytest_add_cli_args", ()) + return tuple(str(arg) for arg in args) if isinstance(args, list) else () + + +def describe(scope: Scope, max_functions: int) -> str: + def section(title: str, values: Iterable[str]) -> Iterator[str]: + listed = tuple(values) + yield f"{title}: {len(listed)}" + yield from (f" {value}" for value in listed) + + lines: Final = ( + *section("changed production files", (f"{f.path} ({len(f.lines)} lines)" for f in scope.files)), + *section("test selection", scope.tests), + *section("functions to mutate", scope.globs), + *( + section(f"NOT mutated (over --max-functions={max_functions})", scope.dropped_globs) + if scope.dropped_globs + else () + ), + ) + return "\n".join(lines) + + +def emit_github_output(scope: Scope) -> None: + if (output := os.environ.get("GITHUB_OUTPUT")) is None: + return + with open(output, "a", encoding="utf-8") as handle: + handle.write(f"has_scope={'true' if scope.is_runnable else 'false'}\n") + handle.write(f"function_count={len(scope.globs)}\n") + + +def main(argv: Sequence[str]) -> int: + parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", default="origin/litellm_internal_staging") + parser.add_argument("--max-functions", type=int, default=40) + parser.add_argument("--root", type=Path, default=REPO_ROOT) + parser.add_argument("--write-pyproject", action="store_true") + parser.add_argument("--globs-out", type=Path, default=None) + parser.add_argument("--json-out", type=Path, default=None) + args: Final = parser.parse_args(argv) + + root: Final = args.root.resolve() + scope: Final = build_scope(root, args.base, args.max_functions) + globs_out: Final = args.globs_out or root / "mutmut-scope-globs.txt" + json_out: Final = args.json_out or root / "mutmut-scope.json" + + globs_out.write_text("".join(f"{glob}\n" for glob in scope.globs), encoding="utf-8") + json_out.write_text( + json.dumps( + { + "files": [{"path": f.path, "lines": sorted(f.lines)} for f in scope.files], + "tests": list(scope.tests), + "globs": list(scope.globs), + "dropped_globs": list(scope.dropped_globs), + }, + indent=2, + ), + encoding="utf-8", + ) + print(describe(scope, args.max_functions)) + + if args.write_pyproject and scope.is_runnable: + rewrite_pyproject(root, scope, existing_pytest_cli_args(root)) + print("rewrote [tool.mutmut] in pyproject.toml for this diff") + + emit_github_output(scope) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py index a606e3f71cf..86c1cf728b5 100644 --- a/scripts/mutation_report.py +++ b/scripts/mutation_report.py @@ -15,7 +15,9 @@ from __future__ import annotations import ast import json +import os import re +import shlex import subprocess import sys import tomllib @@ -25,7 +27,11 @@ from pathlib import Path from textwrap import dedent ROOT = Path(__file__).resolve().parent.parent -MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"] +# Overridable so the report can be regenerated outside CI, where the project venv +# `uv run --no-sync` expects may not exist. +MUTMUT_INVOCATION = shlex.split( + os.environ.get("MUTMUT_CMD", "uv run --no-sync --with mutmut==3.5.0 mutmut") +) def load_mutmut_config() -> dict: @@ -33,16 +39,42 @@ def load_mutmut_config() -> dict: return tomllib.load(f)["tool"]["mutmut"] -def get_survivors() -> list[str]: +def get_results() -> list[tuple[str, str]]: + """Parse `mutmut results` into (mutant name, status) pairs. + + A diff-scoped run leaves every out-of-scope mutant at `not checked`, so + callers must drop that status before counting anything. + """ proc = subprocess.run( - [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False + [*MUTMUT_INVOCATION, "results", "--all=true"], capture_output=True, text=True, check=False ) - survivors = [] + results = [] for line in proc.stdout.splitlines(): - m = re.match(r"\s*(\S+):\s*survived\s*$", line) + m = re.match(r"\s*(\S+):\s*(killed|survived|no tests|timeout|suspicious|skipped|not checked)\s*$", line) if m: - survivors.append(m.group(1)) - return survivors + results.append((m.group(1), m.group(2))) + return results + + +def summarize(results: list[tuple[str, str]]) -> dict | None: + """Count only the mutants this run actually executed.""" + checked = [status for _, status in results if status != "not checked"] + if not checked: + return None + return { + "total": len(checked), + **{ + key: sum(1 for status in checked if status == label) + for key, label in ( + ("killed", "killed"), + ("survived", "survived"), + ("no_tests", "no tests"), + ("skipped", "skipped"), + ("suspicious", "suspicious"), + ("timeout", "timeout"), + ) + }, + } def get_mutmut_show(mutant_name: str) -> str: @@ -399,15 +431,18 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: def main() -> int: config = load_mutmut_config() - stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" - stats: dict | None = None - if stats_file.exists(): - try: - stats = json.loads(stats_file.read_text()) - except json.JSONDecodeError as exc: - print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + results = get_results() + stats = summarize(results) - survivors = get_survivors() + if stats is None: + stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" + if stats_file.exists(): + try: + stats = json.loads(stats_file.read_text()) + except json.JSONDecodeError as exc: + print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + + survivors = [name for name, status in results if status == "survived"] report = render(config, survivors, stats) out_path = ROOT / "mutation-report.md" diff --git a/tests/test_litellm/test_mutation_diff_scope.py b/tests/test_litellm/test_mutation_diff_scope.py new file mode 100644 index 00000000000..f6ae38a069b --- /dev/null +++ b/tests/test_litellm/test_mutation_diff_scope.py @@ -0,0 +1,242 @@ +"""Regression tests for the diff-scoped mutation testing gate. + +`.github/workflows/mutation-test-pr.yml` runs mutmut on pull requests, and the +only thing keeping that job inside a few minutes is `scripts/mutation_diff_scope.py` +narrowing mutmut down to what the diff touched. Two things have to hold, or the +job either burns hours or silently tests nothing: + +- The mutant-name globs must address the exact functions containing the changed + lines, spelled the way mutmut mangles them (`x_` for a module-level + function, `xǁǁ` for a method). A glob that matches nothing makes + mutmut abort; a glob that is too broad drags in unrelated functions. +- The test selection must stay inside `tests/test_litellm/`. `tests/e2e/` needs a + live proxy and cannot run inside mutmut's `mutants/` sandbox, so pulling one in + would fail the clean-test check and abort the run. + +These tests drive the real scope builder against a throwaway git repository laid +out like this one, so a regression shows up as the wrong mutmut invocation. +""" + +from __future__ import annotations + +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from mutation_diff_scope import ( # noqa: E402 + Scope, + build_scope, + render_config, + rewrite_pyproject, + trampoline_units, +) + +PYPROJECT = """\ +[project] +name = "litellm" + +[tool.mutmut] +paths_to_mutate = [ + "litellm/proxy/management_endpoints/", +] +tests_dir = [ + "tests/test_litellm/proxy/management_endpoints/", +] +pytest_add_cli_args = [ + "-p", "no:retry", +] + +[tool.coverage.run] +branch = true +""" + +ROUTER = '''\ +import functools + + +def _deployment_key(name: str) -> str: + return name.strip() + + +class Router: + def __init__(self, models: list[str]) -> None: + self.models = models + + @functools.cached_property + def get_model_list(self) -> list[str]: + return [m for m in self.models if m] + + async def acompletion(self, model: str) -> str: + return model +''' + +AUTH_CHECKS = '''\ +DEFAULT_ROLE = "user" + + +def can_call_model(model: str, allowed: list[str]) -> bool: + return model in allowed +''' + +TYPES = '''\ +def coerce(value: str) -> str: + return value +''' + + +def _run(root: Path, *args: str) -> None: + subprocess.run(("git", *args), cwd=root, check=True, capture_output=True) + + +def _write(root: Path, path: str, body: str) -> None: + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path.resolve() + _run(root, "init", "-q", "-b", "base") + _run(root, "config", "user.email", "test@example.com") + _run(root, "config", "user.name", "test") + + _write(root, "pyproject.toml", PYPROJECT) + _write(root, "litellm/router.py", ROUTER) + _write(root, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS) + _write(root, "litellm/types/utils.py", TYPES) + _write(root, "tests/test_litellm/test_router.py", "def test_router(): pass\n") + _write(root, "tests/test_litellm/proxy/auth/test_auth_checks.py", "def test_auth(): pass\n") + _write(root, "tests/e2e/test_live_proxy.py", "def test_live(): pass\n") + + _run(root, "add", "-A") + _run(root, "commit", "-qm", "base") + _run(root, "checkout", "-q", "-b", "feature") + return root + + +def _commit(root: Path, message: str = "change") -> None: + _run(root, "add", "-A") + _run(root, "commit", "-qm", message) + + +def scope_of(root: Path, max_functions: int = 40) -> Scope: + return build_scope(root, "base", max_functions) + + +def test_changed_method_maps_to_its_mangled_mutant_glob(repo: Path) -> None: + """A line inside a class method must address that method, class-qualified.""" + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.router.xǁRouterǁget_model_list__mutmut_*",) + + +def test_changed_module_level_function_uses_the_underscore_prefix(repo: Path) -> None: + """Module-level functions mangle to `x_`, not `xǁ...ǁ`.""" + _write(repo, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS.replace("model in allowed", "model.lower() in allowed")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.proxy.auth.auth_checks.x_can_call_model__mutmut_*",) + + +def test_touching_a_decorator_still_selects_the_decorated_function(repo: Path) -> None: + """mutmut mutates the whole decorated function, so its decorator lines belong to it.""" + _write(repo, "litellm/router.py", ROUTER.replace("@functools.cached_property", "@functools.cache")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.router.xǁRouterǁget_model_list__mutmut_*",) + + +def test_module_level_change_selects_no_function(repo: Path) -> None: + """mutmut only trampolines functions, so a module-level constant has nothing to run.""" + _write(repo, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS.replace('"user"', '"internal_user"')) + _commit(repo) + + scope = scope_of(repo) + assert scope.files, "the file must still register as changed" + assert scope.globs == () + assert scope.is_runnable is False + + +def test_type_definitions_are_never_mutated(repo: Path) -> None: + """litellm/types holds declarations; mutating them only produces noise.""" + _write(repo, "litellm/types/utils.py", TYPES.replace("return value", "return value.strip()")) + _commit(repo) + + assert scope_of(repo).files == () + + +def test_test_only_changes_produce_nothing_to_mutate(repo: Path) -> None: + _write(repo, "tests/test_litellm/test_router.py", "def test_router(): assert True\n") + _commit(repo) + + assert scope_of(repo).is_runnable is False + + +def test_test_selection_is_the_mirrored_file_plus_changed_unit_tests(repo: Path) -> None: + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _write(repo, "tests/test_litellm/proxy/auth/test_auth_checks.py", "def test_auth(): assert True\n") + _commit(repo) + + assert scope_of(repo).tests == ( + "tests/test_litellm/proxy/auth/test_auth_checks.py", + "tests/test_litellm/test_router.py", + ) + + +def test_e2e_tests_are_excluded_from_the_selection(repo: Path) -> None: + """tests/e2e needs a live proxy; including one aborts the run at mutmut's clean-test check.""" + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _write(repo, "tests/e2e/test_live_proxy.py", "def test_live(): assert True\n") + _commit(repo) + + assert "tests/e2e/test_live_proxy.py" not in scope_of(repo).tests + + +def test_max_functions_caps_the_run_and_reports_what_it_dropped(repo: Path) -> None: + """Silently truncating would read as full coverage of the diff.""" + _write(repo, "litellm/router.py", ROUTER.replace("return name.strip()", "return name.strip().lower()").replace("if m]", "if m and m.strip()]")) + _commit(repo) + + uncapped = scope_of(repo) + assert len(uncapped.globs) == 2 + + capped = scope_of(repo, max_functions=1) + assert len(capped.globs) == 1 + assert capped.dropped_globs == uncapped.globs[1:] + + +def test_rewritten_pyproject_scopes_mutmut_and_keeps_the_rest_of_the_file(repo: Path) -> None: + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _commit(repo) + + rewrite_pyproject(repo, scope_of(repo), ("-p", "no:retry")) + config = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8")) + + assert config["tool"]["mutmut"]["paths_to_mutate"] == ["litellm/router.py"] + assert config["tool"]["mutmut"]["tests_dir"] == ["tests/test_litellm/test_router.py"] + assert config["tool"]["mutmut"]["mutate_only_covered_lines"] is False + assert config["tool"]["mutmut"]["pytest_add_cli_args"] == ["-p", "no:retry"] + assert config["tool"]["coverage"]["run"]["branch"] is True + assert config["project"]["name"] == "litellm" + + +def test_render_config_copies_what_the_selected_tests_import(repo: Path) -> None: + """mutmut runs from mutants/; a test importing litellm or repo tooling needs those copied in.""" + rendered = render_config(Scope(files=(), tests=(), globs=(), dropped_globs=()), ()) + + assert tomllib.loads(rendered)["tool"]["mutmut"]["also_copy"] == ["litellm/", "scripts/", ".github/"] + + +def test_trampoline_units_skips_nested_functions(repo: Path) -> None: + """mutmut folds a nested def into its parent, so selecting it by name would match nothing.""" + source = "def outer():\n def inner():\n return 1\n\n return inner()\n" + + assert [unit.mangled_name for unit in trampoline_units(source)] == ["x_outer"]