mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
ci: polish pytest report summaries
This commit is contained in:
parent
19dedb5549
commit
58b9c6bfa8
3 changed files with 112 additions and 35 deletions
89
.github/scripts/render_pytest_report.py
vendored
89
.github/scripts/render_pytest_report.py
vendored
|
|
@ -7,6 +7,7 @@ import sys
|
|||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, groupby
|
||||
from os.path import commonpath
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
|
|
@ -43,9 +44,9 @@ def _outcome(row: dict[str, object]) -> Outcome:
|
|||
return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"]))
|
||||
|
||||
|
||||
def _cell(value: str | float) -> str:
|
||||
def _cell(value: str | float, limit: int = 240) -> str:
|
||||
raw: Final = str(value)
|
||||
clipped: Final = raw[:240] + ("..." if len(raw) > 240 else "")
|
||||
clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "")
|
||||
return (
|
||||
html.escape(" ".join(clipped.split()), quote=True)
|
||||
.replace("|", "|")
|
||||
|
|
@ -75,18 +76,36 @@ def _metric_row(row: dict[str, object]) -> str:
|
|||
value: Final = row["value"]
|
||||
if not isinstance(value, (str, int, float)):
|
||||
raise ValueError("invalid metric value")
|
||||
return (
|
||||
"| "
|
||||
+ " | ".join(
|
||||
_cell(value)
|
||||
for value in (
|
||||
_string(row["name"]),
|
||||
value,
|
||||
_string(row["unit"]),
|
||||
_string(row["nodeid"]),
|
||||
)
|
||||
unit: Final = _string(row["unit"])
|
||||
measured: Final = _cell(f"{value} {unit}".strip())
|
||||
formatted: Final = f"<strong>{measured}</strong>" if isinstance(value, (int, float)) else measured
|
||||
return f"| {_cell(_string(row['name']))} | {formatted} |"
|
||||
|
||||
|
||||
def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str:
|
||||
return "\n".join(
|
||||
(
|
||||
"| Metric | Value |",
|
||||
"| --- | --- |",
|
||||
_limited(tuple(_metric_row(row) for row in rows)),
|
||||
"",
|
||||
"<details>",
|
||||
"<summary>Metric source</summary>",
|
||||
"",
|
||||
f"<code>{_cell(nodeid)}</code>",
|
||||
"",
|
||||
"</details>",
|
||||
"",
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
|
||||
|
||||
def _expected_detail(outcome: Outcome) -> str:
|
||||
path, _, test = outcome.nodeid.partition("::")
|
||||
return (
|
||||
f"- **{outcome.category}**: {_cell(test or path)}\n\n"
|
||||
f" {_cell(outcome.reason, 1000)}\n\n"
|
||||
f" <sub>{_cell(path)}</sub>\n"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -104,24 +123,36 @@ def render(data: dict[str, object], title: str) -> str:
|
|||
or "No test outcomes recorded"
|
||||
)
|
||||
files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed")
|
||||
prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "."
|
||||
grouped: Final = groupby(
|
||||
sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0]
|
||||
)
|
||||
file_rows: Final = tuple(_file_row(path, tuple(group)) for path, group in grouped if path in files)
|
||||
table: Final = "\n".join(
|
||||
(
|
||||
"| File | passed | failed | error | skipped | xfailed | xpassed |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
_limited(file_rows),
|
||||
file_rows: Final = tuple(
|
||||
_file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files
|
||||
)
|
||||
table: Final = (
|
||||
"\n".join(
|
||||
(
|
||||
"### Files with non-pass outcomes",
|
||||
"",
|
||||
f"Paths relative to <code>{_cell(prefix)}/</code>\n" if prefix != "." else "",
|
||||
"| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
_limited(file_rows),
|
||||
)
|
||||
)
|
||||
if file_rows
|
||||
else ""
|
||||
)
|
||||
metric_groups: Final = groupby(
|
||||
sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"])
|
||||
)
|
||||
metric_table: Final = (
|
||||
"\n".join(
|
||||
(
|
||||
"### Metrics",
|
||||
"",
|
||||
"| Metric | Value | Unit | Test |",
|
||||
"| --- | --- | --- | --- |",
|
||||
_limited(tuple(_metric_row(row) for row in metrics)),
|
||||
_limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)),
|
||||
)
|
||||
)
|
||||
if metrics
|
||||
|
|
@ -138,14 +169,9 @@ def render(data: dict[str, object], title: str) -> str:
|
|||
(
|
||||
"",
|
||||
"<details>",
|
||||
"<summary>Xfail and xpass reasons</summary>",
|
||||
f"<summary>Xfail and xpass reasons ({len(expected)})</summary>",
|
||||
"",
|
||||
_limited(
|
||||
tuple(
|
||||
f"- {outcome.category}: {_cell(outcome.nodeid)}: {_cell(outcome.reason)}"
|
||||
for outcome in expected
|
||||
)
|
||||
),
|
||||
_limited(tuple(_expected_detail(outcome) for outcome in expected)),
|
||||
"",
|
||||
"</details>",
|
||||
)
|
||||
|
|
@ -153,7 +179,10 @@ def render(data: dict[str, object], title: str) -> str:
|
|||
if expected
|
||||
else ""
|
||||
)
|
||||
return "\n".join((f"## {_cell(title)}", "", totals, "", table, metric_table, details, ""))
|
||||
return (
|
||||
"\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, worke
|
|||
)
|
||||
assert ", ".join(terminal) in rendered.stdout
|
||||
assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout)
|
||||
assert "Paths relative to" not in html.unescape(rendered.stdout)
|
||||
assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug")
|
||||
|
||||
|
||||
|
|
@ -173,9 +174,7 @@ def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, worke
|
|||
terminal_summary: Final = next(
|
||||
line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line)
|
||||
)
|
||||
terminal_counts: Final = re.findall(
|
||||
r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary
|
||||
)
|
||||
terminal_counts: Final = re.findall(r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary)
|
||||
assert terminal_counts == ["1 error"]
|
||||
rendered: Final = render(report)
|
||||
assert rendered.returncode == 0, rendered.stderr
|
||||
|
|
@ -264,13 +263,59 @@ def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str)
|
|||
)
|
||||
result: Final = render(report)
|
||||
assert result.returncode == 0
|
||||
visible: Final = result.stdout.count("- xfailed:")
|
||||
visible: Final = result.stdout.count("- **xfailed**:")
|
||||
assert visible <= 500
|
||||
assert f"and {600 - visible} more" in result.stdout
|
||||
assert len(result.stdout.encode()) < 1_000_000
|
||||
assert "<tag>" not in result.stdout
|
||||
|
||||
|
||||
def test_passing_metrics_are_compact_and_preserve_sources(tmp_path: Path) -> None:
|
||||
report: Final = tmp_path / "report.json"
|
||||
_ = report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"outcomes": [{"nodeid": "tests/wheel/test_build.py::test_sizes", "category": "passed", "reason": ""}],
|
||||
"metrics": [
|
||||
{"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Wheel", "value": 20.5, "unit": "MB"},
|
||||
{"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Native", "value": 30, "unit": "MB"},
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
rendered: Final = render(report)
|
||||
assert rendered.returncode == 0, rendered.stderr
|
||||
readable: Final = html.unescape(rendered.stdout)
|
||||
assert "**1 passed**" in readable
|
||||
assert "| Wheel | <strong>20.5 MB</strong> |" in readable
|
||||
assert "| Native | <strong>30 MB</strong> |" in readable
|
||||
assert readable.count("tests/wheel/test_build.py::test_sizes") == 1
|
||||
assert "Files with non-pass outcomes" not in readable
|
||||
|
||||
|
||||
def test_file_rows_shorten_shared_directory_without_losing_paths(tmp_path: Path) -> None:
|
||||
report: Final = tmp_path / "report.json"
|
||||
_ = report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"metrics": [],
|
||||
"outcomes": [
|
||||
{"nodeid": "tests/unit/proxy/test_first.py::test_one", "category": "skipped", "reason": ""},
|
||||
{"nodeid": "tests/unit/proxy/test_second.py::test_two", "category": "failed", "reason": ""},
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
rendered: Final = render(report)
|
||||
readable: Final = html.unescape(rendered.stdout)
|
||||
assert rendered.returncode == 0, rendered.stderr
|
||||
assert "Paths relative to <code>tests/unit/proxy/</code>" in readable
|
||||
assert "| test_first.py | 0 | 0 | 0 | 1 | 0 | 0 |" in readable
|
||||
assert "| test_second.py | 0 | 1 | 0 | 0 | 0 | 0 |" in readable
|
||||
|
||||
|
||||
def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None:
|
||||
result: Final = run_suite(
|
||||
tmp_path,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -28,7 +29,9 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl
|
|||
shim_dir.mkdir()
|
||||
(shim_dir / "uv").write_text(_UV_SHIM)
|
||||
(shim_dir / "uv").chmod(0o755)
|
||||
(tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n")
|
||||
(tmp_path / "pyproject.toml").write_text(
|
||||
f"[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\npythonpath = [{json.dumps(str(_REPO_ROOT / 'scripts'))}]\n"
|
||||
)
|
||||
return subprocess.run(
|
||||
("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()),
|
||||
cwd=tmp_path,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue