From 58b9c6bfa8afba6c1620458fc6d1c13a84fc7dc6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:46:46 +0000 Subject: [PATCH] ci: polish pytest report summaries --- .github/scripts/render_pytest_report.py | 89 ++++++++++++------- .../test_pytest_ci_report.py | 53 ++++++++++- tests/unit/test_unit_shard_missing_paths.py | 5 +- 3 files changed, 112 insertions(+), 35 deletions(-) diff --git a/.github/scripts/render_pytest_report.py b/.github/scripts/render_pytest_report.py index 7da11445643..5aa51fb5cdc 100644 --- a/.github/scripts/render_pytest_report.py +++ b/.github/scripts/render_pytest_report.py @@ -7,6 +7,7 @@ import sys from collections import Counter from dataclasses import dataclass from itertools import accumulate, groupby +from os.path import commonpath from pathlib import Path from typing import Final, cast @@ -43,9 +44,9 @@ def _outcome(row: dict[str, object]) -> Outcome: return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"])) -def _cell(value: str | float) -> str: +def _cell(value: str | float, limit: int = 240) -> str: raw: Final = str(value) - clipped: Final = raw[:240] + ("..." if len(raw) > 240 else "") + clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "") return ( html.escape(" ".join(clipped.split()), quote=True) .replace("|", "|") @@ -75,18 +76,36 @@ def _metric_row(row: dict[str, object]) -> str: value: Final = row["value"] if not isinstance(value, (str, int, float)): raise ValueError("invalid metric value") - return ( - "| " - + " | ".join( - _cell(value) - for value in ( - _string(row["name"]), - value, - _string(row["unit"]), - _string(row["nodeid"]), - ) + unit: Final = _string(row["unit"]) + measured: Final = _cell(f"{value} {unit}".strip()) + formatted: Final = f"{measured}" if isinstance(value, (int, float)) else measured + return f"| {_cell(_string(row['name']))} | {formatted} |" + + +def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str: + return "\n".join( + ( + "| Metric | Value |", + "| --- | --- |", + _limited(tuple(_metric_row(row) for row in rows)), + "", + "
", + "Metric source", + "", + f"{_cell(nodeid)}", + "", + "
", + "", ) - + " |" + ) + + +def _expected_detail(outcome: Outcome) -> str: + path, _, test = outcome.nodeid.partition("::") + return ( + f"- **{outcome.category}**: {_cell(test or path)}\n\n" + f" {_cell(outcome.reason, 1000)}\n\n" + f" {_cell(path)}\n" ) @@ -104,24 +123,36 @@ def render(data: dict[str, object], title: str) -> str: or "No test outcomes recorded" ) files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed") + prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "." grouped: Final = groupby( sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0] ) - file_rows: Final = tuple(_file_row(path, tuple(group)) for path, group in grouped if path in files) - table: Final = "\n".join( - ( - "| File | passed | failed | error | skipped | xfailed | xpassed |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", - _limited(file_rows), + file_rows: Final = tuple( + _file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files + ) + table: Final = ( + "\n".join( + ( + "### Files with non-pass outcomes", + "", + f"Paths relative to {_cell(prefix)}/\n" if prefix != "." else "", + "| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + _limited(file_rows), + ) ) + if file_rows + else "" + ) + metric_groups: Final = groupby( + sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"]) ) metric_table: Final = ( "\n".join( ( + "### Metrics", "", - "| Metric | Value | Unit | Test |", - "| --- | --- | --- | --- |", - _limited(tuple(_metric_row(row) for row in metrics)), + _limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)), ) ) if metrics @@ -138,14 +169,9 @@ def render(data: dict[str, object], title: str) -> str: ( "", "
", - "Xfail and xpass reasons", + f"Xfail and xpass reasons ({len(expected)})", "", - _limited( - tuple( - f"- {outcome.category}: {_cell(outcome.nodeid)}: {_cell(outcome.reason)}" - for outcome in expected - ) - ), + _limited(tuple(_expected_detail(outcome) for outcome in expected)), "", "
", ) @@ -153,7 +179,10 @@ def render(data: dict[str, object], title: str) -> str: if expected else "" ) - return "\n".join((f"## {_cell(title)}", "", totals, "", table, metric_table, details, "")) + return ( + "\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part) + + "\n" + ) def main() -> None: diff --git a/tests/code_coverage_tests/test_pytest_ci_report.py b/tests/code_coverage_tests/test_pytest_ci_report.py index fdd72478998..ec442b9a711 100644 --- a/tests/code_coverage_tests/test_pytest_ci_report.py +++ b/tests/code_coverage_tests/test_pytest_ci_report.py @@ -149,6 +149,7 @@ def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, worke ) assert ", ".join(terminal) in rendered.stdout assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout) + assert "Paths relative to" not in html.unescape(rendered.stdout) assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug") @@ -173,9 +174,7 @@ def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, worke terminal_summary: Final = next( line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line) ) - terminal_counts: Final = re.findall( - r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary - ) + terminal_counts: Final = re.findall(r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary) assert terminal_counts == ["1 error"] rendered: Final = render(report) assert rendered.returncode == 0, rendered.stderr @@ -264,13 +263,59 @@ def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str) ) result: Final = render(report) assert result.returncode == 0 - visible: Final = result.stdout.count("- xfailed:") + visible: Final = result.stdout.count("- **xfailed**:") assert visible <= 500 assert f"and {600 - visible} more" in result.stdout assert len(result.stdout.encode()) < 1_000_000 assert "" not in result.stdout +def test_passing_metrics_are_compact_and_preserve_sources(tmp_path: Path) -> None: + report: Final = tmp_path / "report.json" + _ = report.write_text( + json.dumps( + { + "version": 1, + "outcomes": [{"nodeid": "tests/wheel/test_build.py::test_sizes", "category": "passed", "reason": ""}], + "metrics": [ + {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Wheel", "value": 20.5, "unit": "MB"}, + {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Native", "value": 30, "unit": "MB"}, + ], + } + ) + ) + rendered: Final = render(report) + assert rendered.returncode == 0, rendered.stderr + readable: Final = html.unescape(rendered.stdout) + assert "**1 passed**" in readable + assert "| Wheel | 20.5 MB |" in readable + assert "| Native | 30 MB |" in readable + assert readable.count("tests/wheel/test_build.py::test_sizes") == 1 + assert "Files with non-pass outcomes" not in readable + + +def test_file_rows_shorten_shared_directory_without_losing_paths(tmp_path: Path) -> None: + report: Final = tmp_path / "report.json" + _ = report.write_text( + json.dumps( + { + "version": 1, + "metrics": [], + "outcomes": [ + {"nodeid": "tests/unit/proxy/test_first.py::test_one", "category": "skipped", "reason": ""}, + {"nodeid": "tests/unit/proxy/test_second.py::test_two", "category": "failed", "reason": ""}, + ], + } + ) + ) + rendered: Final = render(report) + readable: Final = html.unescape(rendered.stdout) + assert rendered.returncode == 0, rendered.stderr + assert "Paths relative to tests/unit/proxy/" in readable + assert "| test_first.py | 0 | 0 | 0 | 1 | 0 | 0 |" in readable + assert "| test_second.py | 0 | 1 | 0 | 0 | 0 | 0 |" in readable + + def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None: result: Final = run_suite( tmp_path, diff --git a/tests/unit/test_unit_shard_missing_paths.py b/tests/unit/test_unit_shard_missing_paths.py index 725551e19d3..60cc1390833 100644 --- a/tests/unit/test_unit_shard_missing_paths.py +++ b/tests/unit/test_unit_shard_missing_paths.py @@ -1,3 +1,4 @@ +import json import os import subprocess import sys @@ -28,7 +29,9 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl shim_dir.mkdir() (shim_dir / "uv").write_text(_UV_SHIM) (shim_dir / "uv").chmod(0o755) - (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n") + (tmp_path / "pyproject.toml").write_text( + f"[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\npythonpath = [{json.dumps(str(_REPO_ROOT / 'scripts'))}]\n" + ) return subprocess.run( ("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()), cwd=tmp_path,