diff --git a/pyproject.toml b/pyproject.toml index 6e3c181ae1d..57df956bdc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -341,9 +341,13 @@ filterwarnings = [ paths_to_mutate = [ "litellm/proxy/management_endpoints/", ] +# Only the unit tier that maps to paths_to_mutate. mutmut times and +# coverage-maps this whole set once before mutating, so a tier that needs a +# seeded database (tests/proxy_behavior/) kills the run before it starts, and +# a mutation score is only meaningful against the tests that claim to cover +# the mutated code anyway. tests_dir = [ "tests/test_litellm/proxy/management_endpoints/", - "tests/proxy_behavior/management/", ] also_copy = [ "litellm/", @@ -360,10 +364,16 @@ mutate_only_covered_lines = true # - rerunning a "failed" test on a mutant would mask which mutants are killed # vs. survive, so reruns are wrong for mutation testing regardless. # - xdist is unnecessary inside mutmut (mutmut handles its own parallelism). +# test_saml_sso.py cannot run inside mutmut's mutants/ sandbox: the copied tree +# re-imports cryptography's hash classes under a second identity, so x509 .sign() +# rejects the SHA256 instance the fixture builds with "Algorithm must be a +# registered hash algorithm". Nothing to do with mutation coverage, and one +# erroring test is enough to end the stats phase before any mutant runs. pytest_add_cli_args = [ "-p", "no:retry", "-p", "no:rerunfailures", "-p", "no:xdist", + "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py", ] [tool.coverage.run] diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py index a606e3f71cf..e0d4d569484 100644 --- a/scripts/mutation_report.py +++ b/scripts/mutation_report.py @@ -22,6 +22,7 @@ import tomllib from collections import defaultdict from difflib import SequenceMatcher from pathlib import Path +from typing import Final, NamedTuple from textwrap import dedent ROOT = Path(__file__).resolve().parent.parent @@ -33,16 +34,24 @@ def load_mutmut_config() -> dict: return tomllib.load(f)["tool"]["mutmut"] -def get_survivors() -> list[str]: +class MutmutResults(NamedTuple): + survivors: tuple[str, ...] + reported: int + + +def get_survivors() -> MutmutResults: proc = subprocess.run( [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False ) - survivors = [] - for line in proc.stdout.splitlines(): - m = re.match(r"\s*(\S+):\s*survived\s*$", line) - if m: - survivors.append(m.group(1)) - return survivors + verdicts = tuple( + m.groups() + for line in proc.stdout.splitlines() + if (m := re.match(r"\s*(\S+):\s*(\S.*?)\s*$", line)) + ) + return MutmutResults( + survivors=tuple(name for name, verdict in verdicts if verdict == "survived"), + reported=len(verdicts), + ) def get_mutmut_show(mutant_name: str) -> str: @@ -222,7 +231,52 @@ def render_meta_style_mutant( return "\n".join(out) -def render(config: dict, survivors: list[str], stats: dict | None) -> str: +RESOLVED_KEYS: Final = frozenset({"killed", "survived", "total"}) + + +def unresolved_counts(stats: dict) -> dict[str, int]: + """Every non-zero count that is neither a kill nor a survivor means a mutant did not + reach the tests. Reading it as "anything else" rather than as a list of known statuses + keeps a status this reporter has never met from passing as a clean sweep.""" + return {k: v for k, v in sorted(stats.items()) if k not in RESOLVED_KEYS and isinstance(v, int) and v > 0} + + +def clean_sweep_is_provable(stats: dict | None) -> bool: + """`mutmut results` omits killed mutants, so its silence is equally consistent with a + perfect run and with a run that never started. Only the stats file can tell them apart, + and only when it agrees that nothing survived and every mutant reached the tests.""" + if not stats or stats.get("killed", 0) <= 0 or stats.get("survived", 0) != 0: + return False + return not unresolved_counts(stats) + + +def no_survivors_verdict(results: MutmutResults, stats: dict | None) -> str: + if clean_sweep_is_provable(stats): + return "**No surviving mutants, and the run killed some, so the test suite caught every mutation.**" + if stats and stats.get("survived", 0) > 0: + return ( + f"**mutmut-cicd-stats.json counts {stats['survived']} surviving mutant(s) that " + "`mutmut results` did not list, so the two disagree and neither can be trusted. " + "This is not a passing score.**" + ) + if stats and unresolved_counts(stats): + unresolved = ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in unresolved_counts(stats).items()) + return ( + f"**No survivors, but {unresolved}, so those mutants never reached the tests " + "and the suite was not shown to catch them. This is not a passing score.**" + ) + if stats: + return "**Not one mutant was killed. This is not a passing score.**" + return ( + f"**mutmut-cicd-stats.json is missing and `mutmut results` printed {results.reported} " + "verdict(s), none of them a survivor. Since that command never lists killed mutants, a " + "clean sweep and a run that mutated nothing look identical from here. This is not a " + "passing score.**" + ) + + +def render(config: dict, results: MutmutResults, stats: dict | None) -> str: + survivors = list(results.survivors) by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) for survivor in survivors: module_path, function_name, mutant_num = parse_mutant_name(survivor) @@ -235,17 +289,8 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: out.append("## Summary") out.append("") if stats: - total = stats.get("total", 0) or sum( - stats.get(k, 0) - for k in ( - "killed", - "survived", - "no_tests", - "skipped", - "suspicious", - "timeout", - "segfault", - ) + total = stats.get("total", 0) or ( + stats.get("killed", 0) + stats.get("survived", 0) + sum(unresolved_counts(stats).values()) ) killed = stats.get("killed", 0) survived = stats.get("survived", 0) @@ -254,17 +299,15 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: out.append(f"- Killed: **{killed}**") out.append(f"- Survived: **{survived}**") out.append(f"- Mutation score: **{score:.1f}%**") - for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"): - v = stats.get(k, 0) - if v: - out.append(f"- {k.replace('_', ' ').title()}: {v}") + for k, v in unresolved_counts(stats).items(): + out.append(f"- {k.replace('_', ' ').title()}: {v}") else: out.append(f"- Survivors found: **{len(survivors)}**") out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)") out.append("") if not survivors: - out.append("**No surviving mutants — the test suite caught every mutation.**") + out.append(no_survivors_verdict(results, stats)) out.append("") return "\n".join(out) @@ -407,15 +450,22 @@ def main() -> int: except json.JSONDecodeError as exc: print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) - survivors = get_survivors() - report = render(config, survivors, stats) + results = get_survivors() + report = render(config, results, stats) out_path = ROOT / "mutation-report.md" out_path.write_text(report) print( - f"Wrote {out_path} ({len(survivors)} survivor" - f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)" + f"Wrote {out_path} ({len(results.survivors)} survivor" + f"{'s' if len(results.survivors) != 1 else ''}, {len(report)} chars)" ) + if not results.survivors and not clean_sweep_is_provable(stats): + print( + "error: nothing was shown to have been killed, so the report cannot say " + "anything about the suite", + file=sys.stderr, + ) + return 1 return 0 diff --git a/tests/test_litellm/test_mutation_report.py b/tests/test_litellm/test_mutation_report.py new file mode 100644 index 00000000000..60b29ef2628 --- /dev/null +++ b/tests/test_litellm/test_mutation_report.py @@ -0,0 +1,139 @@ +"""Tests for scripts/mutation_report.py. + +The report is the only thing anyone reads after a mutation run, so the one thing it +must never do is describe a run that produced nothing as a run that killed everything. +`render` decides that wording and `get_survivors` supplies the evidence for it, so both +are tested directly. +""" + +import importlib.util +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_MODULE_PATH = _REPO_ROOT / "scripts" / "mutation_report.py" +_spec = importlib.util.spec_from_file_location("mutation_report", _MODULE_PATH) +report = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = report +_spec.loader.exec_module(report) + +_CONFIG = {"paths_to_mutate": ["litellm/proxy/management_endpoints/"], "tests_dir": ["tests/"]} + + +def test_a_run_that_reported_nothing_is_not_a_clean_sweep(): + rendered = report.render(_CONFIG, report.MutmutResults(survivors=(), reported=0), None) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_a_run_that_killed_every_mutant_says_so(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=0), {"killed": 48, "survived": 0} + ) + + assert "caught every mutation" in rendered + assert "not a passing score" not in rendered + + +def test_stats_counting_survivors_results_never_listed_is_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=0), {"killed": 48, "survived": 3} + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "3 surviving mutant(s)" in rendered + + +def test_mutants_that_never_reached_the_tests_are_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, + report.MutmutResults(survivors=(), reported=0), + {"killed": 48, "survived": 0, "no_tests": 4, "timeout": 1}, + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "4 no tests" in rendered + assert "1 timeout" in rendered + + +def test_a_status_the_reporter_has_never_met_still_blocks_a_clean_sweep(): + rendered = report.render( + _CONFIG, + report.MutmutResults(survivors=(), reported=0), + {"killed": 48, "survived": 0, "check_was_interrupted_by_user": 2}, + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "2 check was interrupted by user" in rendered + + +def test_no_survivors_without_a_kill_is_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=48), {"killed": 0, "survived": 0} + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_no_survivors_and_no_stats_cannot_claim_a_sweep(): + """`mutmut results` never lists killed mutants, so with the stats file missing an + empty survivor list is equally consistent with a perfect run and a dead one.""" + rendered = report.render(_CONFIG, report.MutmutResults(survivors=(), reported=48), None) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_survivors_are_read_out_of_the_verdicts_they_came_with(monkeypatch): + class _Proc: + stdout = ( + "litellm.proxy.management_endpoints.key_management_endpoints.x_1: killed\n" + "litellm.proxy.management_endpoints.key_management_endpoints.x_2: survived\n" + "litellm.proxy.management_endpoints.key_management_endpoints.x_3: no tests\n" + "not a verdict line at all\n" + ) + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + results = report.get_survivors() + + assert results.survivors == ( + "litellm.proxy.management_endpoints.key_management_endpoints.x_2", + ) + assert results.reported == 3 + + +def test_every_multi_word_verdict_mutmut_can_emit_still_counts(monkeypatch): + class _Proc: + stdout = "".join( + f"litellm.proxy.management_endpoints.key_management_endpoints.x_{i}: {verdict}\n" + for i, verdict in enumerate( + ( + "no tests", + "not checked", + "caught by type check", + "check was interrupted by user", + ) + ) + ) + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + results = report.get_survivors() + + assert results.survivors == () + assert results.reported == 4 + + +def test_an_empty_mutmut_results_reports_nothing_rather_than_zero_survivors(monkeypatch): + class _Proc: + stdout = "" + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + assert report.get_survivors() == report.MutmutResults(survivors=(), reported=0)