From c2098389b2f0e1fe82cf3743188c406a6f73bea6 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Sat, 9 May 2026 18:27:11 -0700 Subject: [PATCH] ci(mutation): generate ACH-style report with embedded function bodies Replaces the inline bash markdown generation with a Python script that: - Groups survivors by function (one section per function, function body shown once per section, surviving mutants nested as subsections) - Embeds each enclosing function's source via Python AST (so the agent has full context, not just a 3-line `mutmut show` diff) - Inlines the existing test file(s) listed in [tool.mutmut].tests_dir - Writes an ACH-style task description at the bottom following the prompt template from arXiv 2501.12862 Output goes to mutation-report.md (artifact) and the head of the file is appended to $GITHUB_STEP_SUMMARY for at-a-glance visibility. --- .github/workflows/mutation-test.yml | 53 ++---- scripts/mutation_report.py | 261 ++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 40 deletions(-) create mode 100644 scripts/mutation_report.py diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index d478e13bb17..077a8e448f3 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -97,54 +97,27 @@ jobs: mkdir -p mutants uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log - # Build a detailed report: per-survivor diffs from `mutmut show`, plus - # the killed/survived/total summary from `mutmut export-cicd-stats`. - # Written to both an artifact and the run-page summary so results are - # visible without downloading anything. + # Generate the structured report. The script embeds the enclosing + # function source for each survivor (via Python AST) and includes the + # existing test files, so an LLM agent has enough context to write + # killing tests without further file lookups. Modeled on Meta's ACH + # prompt template (arXiv 2501.12862). - name: Generate detailed mutation report if: always() run: | set +e uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1 uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1 - + uv run --no-sync python scripts/mutation_report.py + # The full report can be very long for big test files; the run-page + # summary cuts off at 1 MB. Append the head of the report (summary + # + survivor list) and link out to the artifact for the full body. { - echo "# Mutation Test Report" + head -c 900000 mutation-report.md echo "" - echo "## Summary" echo "" - if [ -f mutants/mutmut-cicd-stats.json ]; then - echo '```json' - cat mutants/mutmut-cicd-stats.json - echo '```' - else - echo "(mutmut-cicd-stats.json not generated)" - fi - echo "" - echo "## Run progress (tail)" - echo "" - echo '```' - tail -30 mutmut-run.log || true - echo '```' - echo "" - echo "## Surviving mutants (test gaps)" - echo "" - survivors=$(awk '/survived/ {gsub(/:$/,"",$1); print $1}' mutmut-results.txt) - if [ -z "$survivors" ]; then - echo "_None — every mutant was killed by the test suite._" - else - for m in $survivors; do - echo "### \`$m\`" - echo "" - echo '```diff' - uv run --no-sync --with mutmut==3.5.0 mutmut show "$m" 2>&1 || echo "(mutmut show failed for $m)" - echo '```' - echo "" - done - fi - } > mutmut-report.md - - cat mutmut-report.md >> "$GITHUB_STEP_SUMMARY" + echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._" + } >> "$GITHUB_STEP_SUMMARY" - name: Upload mutmut artifacts if: always() @@ -152,7 +125,7 @@ jobs: with: name: mutmut-${{ github.run_id }}-${{ github.run_attempt }} path: | - mutmut-report.md + mutation-report.md mutmut-results.txt mutmut-run.log mutants/mutmut-stats.json diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py new file mode 100644 index 00000000000..99cba7fdec9 --- /dev/null +++ b/scripts/mutation_report.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Generate an agent-actionable mutation testing report. + +Reads the mutmut sandbox state at `mutants/` and produces a single +`mutation-report.md` grouped by function. For each function with surviving +mutants, the report embeds the original function source (via AST), the +unified diff for each surviving mutation (via `mutmut show`), and the +existing test file(s) — followed by an ACH-style instruction asking the +reader to write tests that kill the survivors. + +Run after `mutmut run` and `mutmut export-cicd-stats`. Expects mutmut to be +invokable as `uv run --no-sync --with mutmut== mutmut `. +""" +from __future__ import annotations + +import ast +import json +import re +import subprocess +import sys +import tomllib +from collections import defaultdict +from pathlib import Path +from textwrap import dedent + +ROOT = Path(__file__).resolve().parent.parent +MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"] + + +def load_mutmut_config() -> dict: + with open(ROOT / "pyproject.toml", "rb") as f: + return tomllib.load(f)["tool"]["mutmut"] + + +def get_survivors() -> list[str]: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False + ) + survivors = [] + for line in proc.stdout.splitlines(): + m = re.match(r"\s*(\S+):\s*survived\s*$", line) + if m: + survivors.append(m.group(1)) + return survivors + + +def get_mutmut_show(mutant_name: str) -> str: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "show", mutant_name], + capture_output=True, + text=True, + check=False, + ) + return proc.stdout.strip() or "(mutmut show produced no output)" + + +def parse_mutant_name(name: str) -> tuple[str, str, str]: + """Parse `.x____mutmut_` -> (module, function, N).""" + m = re.match(r"^(.+)\.x__(.+)__mutmut_(\d+)$", name) + if not m: + return name, name, "?" + return m.group(1), m.group(2), m.group(3) + + +def module_to_file(module_path: str) -> Path | None: + candidate = ROOT / Path(*module_path.split(".")).with_suffix(".py") + return candidate if candidate.exists() else None + + +def find_function_in_file( + file_path: Path, function_name: str +) -> tuple[int, int, str] | None: + src = file_path.read_text() + tree = ast.parse(src) + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == function_name + ): + lines = src.splitlines() + return ( + node.lineno, + node.end_lineno, + "\n".join(lines[node.lineno - 1 : node.end_lineno]), + ) + return None + + +def collect_test_files(tests_dir: list[str]) -> list[Path]: + found: list[Path] = [] + for entry in tests_dir: + p = ROOT / entry + if p.is_file(): + found.append(p) + elif p.is_dir(): + found.extend(sorted(p.rglob("test_*.py"))) + return found + + +def render(config: dict, survivors: list[str], stats: dict | None) -> str: + by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + for survivor in survivors: + module_path, function_name, mutant_num = parse_mutant_name(survivor) + by_function[(module_path, function_name)].append((survivor, mutant_num)) + + out: list[str] = [] + out.append("# Mutation Test Report") + out.append("") + + out.append("## Summary") + out.append("") + if stats: + total = stats.get("total", 0) or sum( + stats.get(k, 0) + for k in ( + "killed", + "survived", + "no_tests", + "skipped", + "suspicious", + "timeout", + "segfault", + ) + ) + killed = stats.get("killed", 0) + survived = stats.get("survived", 0) + score = (killed / total * 100) if total else 0.0 + out.append(f"- Total mutants: **{total}**") + out.append(f"- Killed: **{killed}**") + out.append(f"- Survived: **{survived}**") + out.append(f"- Mutation score: **{score:.1f}%**") + for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"): + v = stats.get(k, 0) + if v: + out.append(f"- {k.replace('_', ' ').title()}: {v}") + else: + out.append(f"- Survivors found: **{len(survivors)}**") + out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)") + out.append("") + + if not survivors: + out.append("**No surviving mutants — the test suite caught every mutation.**") + out.append("") + return "\n".join(out) + + out.append("## Surviving mutants by function") + out.append("") + for (module_path, function_name), items in by_function.items(): + anchor = f"{function_name.lower().replace('_', '-')}" + out.append( + f"- [`{function_name}`](#{anchor}) — {len(items)} mutant" + f"{'s' if len(items) != 1 else ''} ({module_path})" + ) + out.append("") + + for (module_path, function_name), items in by_function.items(): + out.append(f"## `{function_name}`") + out.append("") + out.append(f"**Module:** `{module_path}`") + + file_path = module_to_file(module_path) + if file_path is None: + out.append("") + out.append(f"_(could not locate source file for module `{module_path}`)_") + out.append("") + else: + rel = file_path.relative_to(ROOT) + out.append(f"**File:** `{rel}`") + out.append("") + found = find_function_in_file(file_path, function_name) + if found: + start, end, fn_src = found + out.append(f"### Original function (lines {start}-{end})") + out.append("") + out.append("```python") + out.append(fn_src) + out.append("```") + out.append("") + else: + out.append(f"_(could not locate `{function_name}` in {rel} via AST)_") + out.append("") + + out.append(f"### Surviving mutations ({len(items)})") + out.append("") + for i, (mutant_name, mutant_num) in enumerate(items, 1): + out.append(f"#### Mutation {i} of {len(items)} — `{mutant_name}`") + out.append("") + out.append("```diff") + out.append(get_mutmut_show(mutant_name)) + out.append("```") + out.append("") + + test_files = collect_test_files(config.get("tests_dir", [])) + if test_files: + out.append("## Existing tests") + out.append("") + out.append( + "These are the test files that mutmut considered when classifying the " + "mutants above. New tests should be added here, matching existing " + "conventions, fixtures, and naming." + ) + out.append("") + for tf in test_files: + rel = tf.relative_to(ROOT) + out.append(f"### `{rel}`") + out.append("") + out.append("```python") + out.append(tf.read_text()) + out.append("```") + out.append("") + + out.append("## Task") + out.append("") + out.append( + dedent( + """\ + For each surviving mutant listed above, write a new test in the + existing test file (matching its conventions, fixtures, and naming + style) that: + + - **Fails** when the mutated version of the function is in place. + - **Passes** when the original (correct) version is in place. + + Aim for one test per surviving mutant. If multiple mutants in the + same function can be killed by a single test, that is fine — note + which mutant numbers in the test name or docstring. + + Do not modify the source file. Only add tests. + """ + ).strip() + ) + out.append("") + + return "\n".join(out) + + +def main() -> int: + config = load_mutmut_config() + + stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" + stats: dict | None = None + if stats_file.exists(): + try: + stats = json.loads(stats_file.read_text()) + except json.JSONDecodeError as exc: + print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + + survivors = get_survivors() + report = render(config, survivors, stats) + + out_path = ROOT / "mutation-report.md" + out_path.write_text(report) + print( + f"Wrote {out_path} ({len(survivors)} survivor" + f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())