diff --git a/.github/workflows/mutation-test-pr.yml b/.github/workflows/mutation-test-pr.yml new file mode 100644 index 00000000000..b7ae605ccd6 --- /dev/null +++ b/.github/workflows/mutation-test-pr.yml @@ -0,0 +1,158 @@ +name: "Mutation Test (PR diff)" + +# Diff-scoped mutation testing, modeled on Google's "State of Mutation Testing +# at Google" (https://research.google/pubs/state-of-mutation-testing-at-google/): +# mutate only what the pull request changed, not the whole codebase. +# +# scripts/mutation_diff_scope.py rewrites [tool.mutmut] so paths_to_mutate is +# the changed production files and tests_dir is the tests mirroring them, then +# emits mutant-name globs for the functions containing the changed lines. +# +# Advisory only: the result is written to the job summary and uploaded as an +# artifact. Nothing here blocks a merge. + +on: + pull_request: + paths: + - "litellm/**.py" + - "scripts/mutation_diff_scope.py" + - ".github/workflows/mutation-test-pr.yml" + +permissions: + contents: read + +concurrency: + group: mutation-test-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + mutation: + name: Mutate the diff + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + # The scope script diffs HEAD against the merge base with the target + # branch, so the full history has to be present. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Scope the run to the diff + id: scope + run: | + python scripts/mutation_diff_scope.py \ + --base "origin/${{ github.event.pull_request.base.ref }}" \ + --write-pyproject | tee mutmut-scope.log + + - name: Set up uv + if: steps.scope.outputs.has_scope == 'true' + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache uv dependencies + if: steps.scope.outputs.has_scope == 'true' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + if: steps.scope.outputs.has_scope == 'true' + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + + - name: Cache Prisma binaries + if: steps.scope.outputs.has_scope == 'true' + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + if: steps.scope.outputs.has_scope == 'true' + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + # See .github/workflows/mutation-test.yml for why these two steps exist: + # an editable install shadows the mutants/ sandbox, and pytest-retry + # crashes under mutmut's in-process pytest.main() call. + - name: Reinstall litellm non-editable (so mutants/ is not shadowed) + if: steps.scope.outputs.has_scope == 'true' + run: | + uv pip uninstall litellm + uv pip install . --no-deps + + - name: Remove pytest plugins that conflict with mutmut + if: steps.scope.outputs.has_scope == 'true' + run: | + uv pip uninstall pytest-retry || true + + - name: Run mutmut on the changed functions + if: steps.scope.outputs.has_scope == 'true' + continue-on-error: true + timeout-minutes: 10 + env: + PYTHONPATH: ${{ github.workspace }}/mutants + run: | + set -o pipefail + # mutants/ has to exist before mutmut starts: it is on PYTHONPATH, and Python + # caches a missing sys.path entry, so a directory created later is never searched. + mkdir -p mutants + GLOBS=() + while IFS= read -r glob || [ -n "$glob" ]; do + [ -n "$glob" ] && GLOBS+=("$glob") + done < mutmut-scope-globs.txt + uv run --no-sync --with mutmut==3.5.0 mutmut run "${GLOBS[@]}" 2>&1 | tee mutmut-run.log + + - name: Generate mutation report + if: always() && steps.scope.outputs.has_scope == 'true' + run: | + set +e + uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1 + uv run --no-sync --with mutmut==3.5.0 mutmut results --all=true > mutmut-results.txt 2>&1 + uv run --no-sync python scripts/mutation_report.py + { + echo "## Mutation testing on this diff" + echo "" + echo '```' + cat mutmut-scope.log + echo '```' + echo "" + head -c 900000 mutation-report.md + } >> "$GITHUB_STEP_SUMMARY" + + - name: Report that nothing was in scope + if: steps.scope.outputs.has_scope != 'true' + run: | + { + echo "## Mutation testing on this diff" + echo "" + echo "No mutable production code changed, or no mirrored tests were found for it." + echo "" + echo '```' + cat mutmut-scope.log + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload mutation artifacts + if: always() && steps.scope.outputs.has_scope == 'true' + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: mutmut-pr-${{ github.run_id }}-${{ github.run_attempt }} + path: | + mutation-report.md + mutmut-results.txt + mutmut-run.log + mutmut-scope.json + mutmut-scope.log + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 201e02f2189..cb1b21a9aef 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,12 @@ crash.*.log ui/litellm-dashboard/out/ litellm.log + +# mutation testing scratch (mutmut sandbox + diff-scope artifacts) +mutants/ +mutation-report.md +mutmut-results.txt +mutmut-run.log +mutmut-scope.json +mutmut-scope-globs.txt +mutmut-scope.log diff --git a/scripts/mutation_diff_scope.py b/scripts/mutation_diff_scope.py new file mode 100644 index 00000000000..4ccc77fceab --- /dev/null +++ b/scripts/mutation_diff_scope.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Scope a mutmut run to the production code a pull request actually changed. + +mutmut reads its configuration only from ``[tool.mutmut]`` in ``pyproject.toml``, +and it filters which mutants to *execute* by fnmatch against mutant names. This +script bridges a git diff to both: + +1. It rewrites ``[tool.mutmut]`` so ``paths_to_mutate`` is the changed production + files and ``tests_dir`` is the test files that mirror them. +2. It emits mutant-name globs for the functions containing the changed lines, to + be passed as arguments to ``mutmut run``. + +mutmut installs its trampolines per top-level function and per class method, so a +changed line's enclosing function is the smallest unit it can execute. + +Usage: + python scripts/mutation_diff_scope.py --base origin/litellm_internal_staging +""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import subprocess +import sys +import tomllib +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent +MUTATE_ROOTS: Final = ("litellm/",) +EXCLUDED_PREFIXES: Final = ( + "litellm/proxy/_experimental/", + "litellm/types/", +) +TEST_MIRROR_ROOT: Final = "tests/test_litellm" +# mutmut runs the selected tests from a `mutants/` sandbox. It copies tests/ itself, but a +# test that reaches back out to repo tooling (scripts/, .github/) fails to import there. +SANDBOX_COPIES: Final = ("litellm/", "scripts/", ".github/") +CLASS_NAME_SEPARATOR: Final = "ǁ" +HUNK_HEADER: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@") +CONFIG_SECTION: Final = re.compile(r"^\[tool\.mutmut\]$.*?(?=^\[)", re.MULTILINE | re.DOTALL) + + +@dataclass(frozen=True, slots=True) +class ChangedFile: + path: str + lines: frozenset[int] + + +@dataclass(frozen=True, slots=True) +class TrampolineUnit: + """A function or method mutmut can address by name, and the lines it spans.""" + + mangled_name: str + start_line: int + end_line: int + + +@dataclass(frozen=True, slots=True) +class Scope: + files: tuple[ChangedFile, ...] + tests: tuple[str, ...] + globs: tuple[str, ...] + dropped_globs: tuple[str, ...] + + @property + def is_runnable(self) -> bool: + return bool(self.globs and self.tests) + + +def _git(root: Path, *args: str) -> str: + return subprocess.run(("git", *args), cwd=root, capture_output=True, text=True, check=True).stdout + + +def is_mutable_path(root: Path, path: str) -> bool: + return ( + path.endswith(".py") + and path.startswith(MUTATE_ROOTS) + and not path.startswith(EXCLUDED_PREFIXES) + and (root / path).is_file() + ) + + +def is_unit_test_path(root: Path, path: str) -> bool: + """Only the mirrored unit suite; tests/e2e needs a live proxy and cannot run in mutmut's sandbox.""" + return ( + path.startswith(f"{TEST_MIRROR_ROOT}/") + and path.endswith(".py") + and Path(path).name.startswith("test_") + and (root / path).is_file() + ) + + +def parse_unified_diff(root: Path, diff: str) -> tuple[ChangedFile, ...]: + """Map each changed production file to the post-image line numbers the diff touched.""" + + def touched_lines() -> Iterator[tuple[str, int]]: + current = "" + cursor = 0 + for line in diff.splitlines(): + if line.startswith("+++ b/"): + current = line[len("+++ b/") :] + elif (header := HUNK_HEADER.match(line)) is not None: + cursor = int(header.group(1)) + elif line.startswith("+") and not line.startswith("+++") and current: + yield current, cursor + cursor += 1 + + touched: Final = tuple(touched_lines()) + return tuple( + ChangedFile(path=path, lines=frozenset(line for other, line in touched if other == path)) + for path in sorted({path for path, _ in touched}) + if is_mutable_path(root, path) + ) + + +def changed_test_files(root: Path, diff: str) -> tuple[str, ...]: + return tuple( + sorted( + path + for line in diff.splitlines() + if line.startswith("+++ b/") and is_unit_test_path(root, path := line[len("+++ b/") :]) + ) + ) + + +def mapped_tests(root: Path, path: str) -> tuple[str, ...]: + """Find the tests mirroring a production file, per tests/test_litellm/readme.md.""" + relative: Final = Path(path).relative_to("litellm") + mirror_dir: Final = Path(TEST_MIRROR_ROOT) / relative.parent + named: Final = tuple(sorted(str(p.relative_to(root)) for p in (root / mirror_dir).glob(f"test_{relative.stem}*.py"))) + if named: + return named + return (str(mirror_dir),) if (root / mirror_dir).is_dir() else () + + +def trampoline_units(source: str) -> tuple[TrampolineUnit, ...]: + """Every unit mutmut trampolines: top-level functions and direct class methods.""" + tree: Final = ast.parse(source) + + def unit(node: ast.stmt, class_name: str | None) -> TrampolineUnit | None: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return None + separator = CLASS_NAME_SEPARATOR + mangled = f"x{separator}{class_name}{separator}{node.name}" if class_name else f"x_{node.name}" + return TrampolineUnit( + mangled_name=mangled, + start_line=min((d.lineno for d in node.decorator_list), default=node.lineno), + end_line=node.end_lineno or node.lineno, + ) + + def walk() -> Iterator[TrampolineUnit]: + for statement in tree.body: + if (function := unit(statement, None)) is not None: + yield function + elif isinstance(statement, ast.ClassDef): + for member in statement.body: + if (method := unit(member, statement.name)) is not None: + yield method + + return tuple(walk()) + + +def module_name(path: str) -> str: + return path[: -len(".py")].replace(os.sep, ".").replace("/", ".") + + +def globs_for(root: Path, changed: ChangedFile) -> tuple[str, ...]: + try: + units = trampoline_units((root / changed.path).read_text(encoding="utf-8")) + except SyntaxError: + return () + module: Final = module_name(changed.path) + return tuple( + f"{module}.{unit.mangled_name}__mutmut_*" + for unit in units + if any(unit.start_line <= line <= unit.end_line for line in changed.lines) + ) + + +def build_scope(root: Path, base: str, max_functions: int) -> Scope: + merge_base: Final = _git(root, "merge-base", base, "HEAD").strip() + diff: Final = _git(root, "diff", "-U0", merge_base, "--", ".") + files: Final = parse_unified_diff(root, diff) + all_globs: Final = tuple(glob for changed in files for glob in globs_for(root, changed)) + tests: Final = tuple( + sorted({*(t for changed in files for t in mapped_tests(root, changed.path)), *changed_test_files(root, diff)}) + ) + return Scope( + files=files, + tests=tests, + globs=all_globs[:max_functions], + dropped_globs=all_globs[max_functions:], + ) + + +def render_config(scope: Scope, pytest_add_cli_args: Sequence[str]) -> str: + def toml_list(values: Iterable[str]) -> str: + return "[\n" + "".join(f' "{value}",\n' for value in values) + "]" + + return ( + "[tool.mutmut]\n" + f"paths_to_mutate = {toml_list(f.path for f in scope.files)}\n" + f"tests_dir = {toml_list(scope.tests)}\n" + f"also_copy = {toml_list(SANDBOX_COPIES)}\n" + # Gathering coverage costs an extra full run of the selected tests, and mutants on + # uncovered lines are cheap here: mutmut marks them "no tests" without running any. + "mutate_only_covered_lines = false\n" + f"pytest_add_cli_args = {toml_list(pytest_add_cli_args)}\n" + "\n" + ) + + +def rewrite_pyproject(root: Path, scope: Scope, pytest_add_cli_args: Sequence[str]) -> None: + pyproject: Final = root / "pyproject.toml" + original: Final = pyproject.read_text(encoding="utf-8") + if CONFIG_SECTION.search(original) is None: + raise SystemExit("Could not find a [tool.mutmut] section to replace in pyproject.toml") + pyproject.write_text( + CONFIG_SECTION.sub(lambda _: render_config(scope, pytest_add_cli_args), original, count=1), encoding="utf-8" + ) + + +def existing_pytest_cli_args(root: Path) -> tuple[str, ...]: + with open(root / "pyproject.toml", "rb") as handle: + section: Mapping[str, object] = tomllib.load(handle)["tool"]["mutmut"] + args: Final = section.get("pytest_add_cli_args", ()) + return tuple(str(arg) for arg in args) if isinstance(args, list) else () + + +def describe(scope: Scope, max_functions: int) -> str: + def section(title: str, values: Iterable[str]) -> Iterator[str]: + listed = tuple(values) + yield f"{title}: {len(listed)}" + yield from (f" {value}" for value in listed) + + lines: Final = ( + *section("changed production files", (f"{f.path} ({len(f.lines)} lines)" for f in scope.files)), + *section("test selection", scope.tests), + *section("functions to mutate", scope.globs), + *( + section(f"NOT mutated (over --max-functions={max_functions})", scope.dropped_globs) + if scope.dropped_globs + else () + ), + ) + return "\n".join(lines) + + +def emit_github_output(scope: Scope) -> None: + if (output := os.environ.get("GITHUB_OUTPUT")) is None: + return + with open(output, "a", encoding="utf-8") as handle: + handle.write(f"has_scope={'true' if scope.is_runnable else 'false'}\n") + handle.write(f"function_count={len(scope.globs)}\n") + + +def main(argv: Sequence[str]) -> int: + parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", default="origin/litellm_internal_staging") + parser.add_argument("--max-functions", type=int, default=40) + parser.add_argument("--root", type=Path, default=REPO_ROOT) + parser.add_argument("--write-pyproject", action="store_true") + parser.add_argument("--globs-out", type=Path, default=None) + parser.add_argument("--json-out", type=Path, default=None) + args: Final = parser.parse_args(argv) + + root: Final = args.root.resolve() + scope: Final = build_scope(root, args.base, args.max_functions) + globs_out: Final = args.globs_out or root / "mutmut-scope-globs.txt" + json_out: Final = args.json_out or root / "mutmut-scope.json" + + globs_out.write_text("".join(f"{glob}\n" for glob in scope.globs), encoding="utf-8") + json_out.write_text( + json.dumps( + { + "files": [{"path": f.path, "lines": sorted(f.lines)} for f in scope.files], + "tests": list(scope.tests), + "globs": list(scope.globs), + "dropped_globs": list(scope.dropped_globs), + }, + indent=2, + ), + encoding="utf-8", + ) + print(describe(scope, args.max_functions)) + + if args.write_pyproject and scope.is_runnable: + rewrite_pyproject(root, scope, existing_pytest_cli_args(root)) + print("rewrote [tool.mutmut] in pyproject.toml for this diff") + + emit_github_output(scope) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py index a606e3f71cf..86c1cf728b5 100644 --- a/scripts/mutation_report.py +++ b/scripts/mutation_report.py @@ -15,7 +15,9 @@ from __future__ import annotations import ast import json +import os import re +import shlex import subprocess import sys import tomllib @@ -25,7 +27,11 @@ from pathlib import Path from textwrap import dedent ROOT = Path(__file__).resolve().parent.parent -MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"] +# Overridable so the report can be regenerated outside CI, where the project venv +# `uv run --no-sync` expects may not exist. +MUTMUT_INVOCATION = shlex.split( + os.environ.get("MUTMUT_CMD", "uv run --no-sync --with mutmut==3.5.0 mutmut") +) def load_mutmut_config() -> dict: @@ -33,16 +39,42 @@ def load_mutmut_config() -> dict: return tomllib.load(f)["tool"]["mutmut"] -def get_survivors() -> list[str]: +def get_results() -> list[tuple[str, str]]: + """Parse `mutmut results` into (mutant name, status) pairs. + + A diff-scoped run leaves every out-of-scope mutant at `not checked`, so + callers must drop that status before counting anything. + """ proc = subprocess.run( - [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False + [*MUTMUT_INVOCATION, "results", "--all=true"], capture_output=True, text=True, check=False ) - survivors = [] + results = [] for line in proc.stdout.splitlines(): - m = re.match(r"\s*(\S+):\s*survived\s*$", line) + m = re.match(r"\s*(\S+):\s*(killed|survived|no tests|timeout|suspicious|skipped|not checked)\s*$", line) if m: - survivors.append(m.group(1)) - return survivors + results.append((m.group(1), m.group(2))) + return results + + +def summarize(results: list[tuple[str, str]]) -> dict | None: + """Count only the mutants this run actually executed.""" + checked = [status for _, status in results if status != "not checked"] + if not checked: + return None + return { + "total": len(checked), + **{ + key: sum(1 for status in checked if status == label) + for key, label in ( + ("killed", "killed"), + ("survived", "survived"), + ("no_tests", "no tests"), + ("skipped", "skipped"), + ("suspicious", "suspicious"), + ("timeout", "timeout"), + ) + }, + } def get_mutmut_show(mutant_name: str) -> str: @@ -399,15 +431,18 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: def main() -> int: config = load_mutmut_config() - stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" - stats: dict | None = None - if stats_file.exists(): - try: - stats = json.loads(stats_file.read_text()) - except json.JSONDecodeError as exc: - print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + results = get_results() + stats = summarize(results) - survivors = get_survivors() + if stats is None: + stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" + if stats_file.exists(): + try: + stats = json.loads(stats_file.read_text()) + except json.JSONDecodeError as exc: + print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + + survivors = [name for name, status in results if status == "survived"] report = render(config, survivors, stats) out_path = ROOT / "mutation-report.md" diff --git a/tests/test_litellm/test_mutation_diff_scope.py b/tests/test_litellm/test_mutation_diff_scope.py new file mode 100644 index 00000000000..f6ae38a069b --- /dev/null +++ b/tests/test_litellm/test_mutation_diff_scope.py @@ -0,0 +1,242 @@ +"""Regression tests for the diff-scoped mutation testing gate. + +`.github/workflows/mutation-test-pr.yml` runs mutmut on pull requests, and the +only thing keeping that job inside a few minutes is `scripts/mutation_diff_scope.py` +narrowing mutmut down to what the diff touched. Two things have to hold, or the +job either burns hours or silently tests nothing: + +- The mutant-name globs must address the exact functions containing the changed + lines, spelled the way mutmut mangles them (`x_` for a module-level + function, `xǁǁ` for a method). A glob that matches nothing makes + mutmut abort; a glob that is too broad drags in unrelated functions. +- The test selection must stay inside `tests/test_litellm/`. `tests/e2e/` needs a + live proxy and cannot run inside mutmut's `mutants/` sandbox, so pulling one in + would fail the clean-test check and abort the run. + +These tests drive the real scope builder against a throwaway git repository laid +out like this one, so a regression shows up as the wrong mutmut invocation. +""" + +from __future__ import annotations + +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from mutation_diff_scope import ( # noqa: E402 + Scope, + build_scope, + render_config, + rewrite_pyproject, + trampoline_units, +) + +PYPROJECT = """\ +[project] +name = "litellm" + +[tool.mutmut] +paths_to_mutate = [ + "litellm/proxy/management_endpoints/", +] +tests_dir = [ + "tests/test_litellm/proxy/management_endpoints/", +] +pytest_add_cli_args = [ + "-p", "no:retry", +] + +[tool.coverage.run] +branch = true +""" + +ROUTER = '''\ +import functools + + +def _deployment_key(name: str) -> str: + return name.strip() + + +class Router: + def __init__(self, models: list[str]) -> None: + self.models = models + + @functools.cached_property + def get_model_list(self) -> list[str]: + return [m for m in self.models if m] + + async def acompletion(self, model: str) -> str: + return model +''' + +AUTH_CHECKS = '''\ +DEFAULT_ROLE = "user" + + +def can_call_model(model: str, allowed: list[str]) -> bool: + return model in allowed +''' + +TYPES = '''\ +def coerce(value: str) -> str: + return value +''' + + +def _run(root: Path, *args: str) -> None: + subprocess.run(("git", *args), cwd=root, check=True, capture_output=True) + + +def _write(root: Path, path: str, body: str) -> None: + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path.resolve() + _run(root, "init", "-q", "-b", "base") + _run(root, "config", "user.email", "test@example.com") + _run(root, "config", "user.name", "test") + + _write(root, "pyproject.toml", PYPROJECT) + _write(root, "litellm/router.py", ROUTER) + _write(root, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS) + _write(root, "litellm/types/utils.py", TYPES) + _write(root, "tests/test_litellm/test_router.py", "def test_router(): pass\n") + _write(root, "tests/test_litellm/proxy/auth/test_auth_checks.py", "def test_auth(): pass\n") + _write(root, "tests/e2e/test_live_proxy.py", "def test_live(): pass\n") + + _run(root, "add", "-A") + _run(root, "commit", "-qm", "base") + _run(root, "checkout", "-q", "-b", "feature") + return root + + +def _commit(root: Path, message: str = "change") -> None: + _run(root, "add", "-A") + _run(root, "commit", "-qm", message) + + +def scope_of(root: Path, max_functions: int = 40) -> Scope: + return build_scope(root, "base", max_functions) + + +def test_changed_method_maps_to_its_mangled_mutant_glob(repo: Path) -> None: + """A line inside a class method must address that method, class-qualified.""" + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.router.xǁRouterǁget_model_list__mutmut_*",) + + +def test_changed_module_level_function_uses_the_underscore_prefix(repo: Path) -> None: + """Module-level functions mangle to `x_`, not `xǁ...ǁ`.""" + _write(repo, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS.replace("model in allowed", "model.lower() in allowed")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.proxy.auth.auth_checks.x_can_call_model__mutmut_*",) + + +def test_touching_a_decorator_still_selects_the_decorated_function(repo: Path) -> None: + """mutmut mutates the whole decorated function, so its decorator lines belong to it.""" + _write(repo, "litellm/router.py", ROUTER.replace("@functools.cached_property", "@functools.cache")) + _commit(repo) + + assert scope_of(repo).globs == ("litellm.router.xǁRouterǁget_model_list__mutmut_*",) + + +def test_module_level_change_selects_no_function(repo: Path) -> None: + """mutmut only trampolines functions, so a module-level constant has nothing to run.""" + _write(repo, "litellm/proxy/auth/auth_checks.py", AUTH_CHECKS.replace('"user"', '"internal_user"')) + _commit(repo) + + scope = scope_of(repo) + assert scope.files, "the file must still register as changed" + assert scope.globs == () + assert scope.is_runnable is False + + +def test_type_definitions_are_never_mutated(repo: Path) -> None: + """litellm/types holds declarations; mutating them only produces noise.""" + _write(repo, "litellm/types/utils.py", TYPES.replace("return value", "return value.strip()")) + _commit(repo) + + assert scope_of(repo).files == () + + +def test_test_only_changes_produce_nothing_to_mutate(repo: Path) -> None: + _write(repo, "tests/test_litellm/test_router.py", "def test_router(): assert True\n") + _commit(repo) + + assert scope_of(repo).is_runnable is False + + +def test_test_selection_is_the_mirrored_file_plus_changed_unit_tests(repo: Path) -> None: + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _write(repo, "tests/test_litellm/proxy/auth/test_auth_checks.py", "def test_auth(): assert True\n") + _commit(repo) + + assert scope_of(repo).tests == ( + "tests/test_litellm/proxy/auth/test_auth_checks.py", + "tests/test_litellm/test_router.py", + ) + + +def test_e2e_tests_are_excluded_from_the_selection(repo: Path) -> None: + """tests/e2e needs a live proxy; including one aborts the run at mutmut's clean-test check.""" + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _write(repo, "tests/e2e/test_live_proxy.py", "def test_live(): assert True\n") + _commit(repo) + + assert "tests/e2e/test_live_proxy.py" not in scope_of(repo).tests + + +def test_max_functions_caps_the_run_and_reports_what_it_dropped(repo: Path) -> None: + """Silently truncating would read as full coverage of the diff.""" + _write(repo, "litellm/router.py", ROUTER.replace("return name.strip()", "return name.strip().lower()").replace("if m]", "if m and m.strip()]")) + _commit(repo) + + uncapped = scope_of(repo) + assert len(uncapped.globs) == 2 + + capped = scope_of(repo, max_functions=1) + assert len(capped.globs) == 1 + assert capped.dropped_globs == uncapped.globs[1:] + + +def test_rewritten_pyproject_scopes_mutmut_and_keeps_the_rest_of_the_file(repo: Path) -> None: + _write(repo, "litellm/router.py", ROUTER.replace("if m]", "if m and m.strip()]")) + _commit(repo) + + rewrite_pyproject(repo, scope_of(repo), ("-p", "no:retry")) + config = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8")) + + assert config["tool"]["mutmut"]["paths_to_mutate"] == ["litellm/router.py"] + assert config["tool"]["mutmut"]["tests_dir"] == ["tests/test_litellm/test_router.py"] + assert config["tool"]["mutmut"]["mutate_only_covered_lines"] is False + assert config["tool"]["mutmut"]["pytest_add_cli_args"] == ["-p", "no:retry"] + assert config["tool"]["coverage"]["run"]["branch"] is True + assert config["project"]["name"] == "litellm" + + +def test_render_config_copies_what_the_selected_tests_import(repo: Path) -> None: + """mutmut runs from mutants/; a test importing litellm or repo tooling needs those copied in.""" + rendered = render_config(Scope(files=(), tests=(), globs=(), dropped_globs=()), ()) + + assert tomllib.loads(rendered)["tool"]["mutmut"]["also_copy"] == ["litellm/", "scripts/", ".github/"] + + +def test_trampoline_units_skips_nested_functions(repo: Path) -> None: + """mutmut folds a nested def into its parent, so selecting it by name would match nothing.""" + source = "def outer():\n def inner():\n return 1\n\n return inner()\n" + + assert [unit.mangled_name for unit in trampoline_units(source)] == ["x_outer"]