mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
206 lines
6.9 KiB
Python
206 lines
6.9 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import html
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from itertools import accumulate, groupby
|
|
from os.path import commonpath
|
|
from pathlib import Path
|
|
from typing import Final, cast
|
|
|
|
TOTAL_ORDER: Final = ("failed", "passed", "skipped", "xfailed", "xpassed", "error")
|
|
COLUMNS: Final = ("passed", "failed", "error", "skipped", "xfailed", "xpassed")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Outcome:
|
|
nodeid: str
|
|
category: str
|
|
reason: str
|
|
|
|
|
|
def _object(value: object) -> dict[str, object]:
|
|
if not isinstance(value, dict):
|
|
raise ValueError("expected an object")
|
|
return cast(dict[str, object], value)
|
|
|
|
|
|
def _array(value: object) -> tuple[dict[str, object], ...]:
|
|
if not isinstance(value, list):
|
|
raise ValueError("expected an array")
|
|
return tuple(_object(row) for row in cast(list[object], value))
|
|
|
|
|
|
def _string(value: object) -> str:
|
|
if not isinstance(value, str):
|
|
raise ValueError("expected a string")
|
|
return value
|
|
|
|
|
|
def _outcome(row: dict[str, object]) -> Outcome:
|
|
return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"]))
|
|
|
|
|
|
def _cell(value: str | float, limit: int = 240) -> str:
|
|
raw: Final = str(value)
|
|
clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "")
|
|
return (
|
|
html.escape(" ".join(clipped.split()), quote=True)
|
|
.replace("|", "|")
|
|
.replace("`", "`")
|
|
.replace("[", "[")
|
|
.replace("]", "]")
|
|
.replace("*", "*")
|
|
.replace("_", "_")
|
|
.replace("\\", "\")
|
|
)
|
|
|
|
|
|
def _limited(rows: tuple[str, ...]) -> str:
|
|
candidates: Final = rows[:500]
|
|
sizes: Final = accumulate(len(row.encode("utf-8")) + 1 for row in candidates)
|
|
visible: Final = tuple(row for row, size in zip(candidates, sizes) if size <= 200_000)
|
|
remainder: Final = len(rows) - len(visible)
|
|
return "\n".join(visible + ((f"\nand {remainder} more",) if remainder else ()))
|
|
|
|
|
|
def _file_row(path: str, outcomes: tuple[Outcome, ...]) -> str:
|
|
counts: Final = Counter(outcome.category for outcome in outcomes)
|
|
return "| " + _cell(path) + " | " + " | ".join(str(counts[column]) for column in COLUMNS) + " |"
|
|
|
|
|
|
def _metric_row(row: dict[str, object]) -> str:
|
|
value: Final = row["value"]
|
|
if not isinstance(value, (str, int, float)):
|
|
raise ValueError("invalid metric value")
|
|
unit: Final = _string(row["unit"])
|
|
measured: Final = _cell(f"{value} {unit}".strip())
|
|
formatted: Final = f"<strong>{measured}</strong>" if isinstance(value, (int, float)) else measured
|
|
return f"| {_cell(_string(row['name']))} | {formatted} |"
|
|
|
|
|
|
def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str:
|
|
return "\n".join(
|
|
(
|
|
"| Metric | Value |",
|
|
"| --- | --- |",
|
|
_limited(tuple(_metric_row(row) for row in rows)),
|
|
"",
|
|
"<details>",
|
|
"<summary>Metric source</summary>",
|
|
"",
|
|
f"<code>{_cell(nodeid)}</code>",
|
|
"",
|
|
"</details>",
|
|
"",
|
|
)
|
|
)
|
|
|
|
|
|
def _expected_detail(outcome: Outcome) -> str:
|
|
path, _, test = outcome.nodeid.partition("::")
|
|
return (
|
|
f"- **{outcome.category}**: {_cell(test or path)}\n\n"
|
|
f" {_cell(outcome.reason, 1000)}\n\n"
|
|
f" <sub>{_cell(path)}</sub>\n"
|
|
)
|
|
|
|
|
|
def render(data: dict[str, object], title: str) -> str:
|
|
outcomes: Final = tuple(_outcome(row) for row in _array(data["outcomes"]))
|
|
metrics: Final = _array(data["metrics"])
|
|
counts: Final = Counter(outcome.category for outcome in outcomes)
|
|
extras: Final = tuple(sorted(set(counts) - set(TOTAL_ORDER)))
|
|
totals: Final = (
|
|
", ".join(
|
|
f"{counts[category]} {'errors' if category == 'error' and counts[category] != 1 else category}"
|
|
for category in TOTAL_ORDER + extras
|
|
if counts[category]
|
|
)
|
|
or "No test outcomes recorded"
|
|
)
|
|
files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed")
|
|
prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "."
|
|
grouped: Final = groupby(
|
|
sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0]
|
|
)
|
|
file_rows: Final = tuple(
|
|
_file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files
|
|
)
|
|
table: Final = (
|
|
"\n".join(
|
|
(
|
|
"### Files with non-pass outcomes",
|
|
"",
|
|
f"Paths relative to <code>{_cell(prefix)}/</code>\n" if prefix != "." else "",
|
|
"| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |",
|
|
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
_limited(file_rows),
|
|
)
|
|
)
|
|
if file_rows
|
|
else ""
|
|
)
|
|
metric_groups: Final = groupby(
|
|
sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"])
|
|
)
|
|
metric_table: Final = (
|
|
"\n".join(
|
|
(
|
|
"### Metrics",
|
|
"",
|
|
_limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)),
|
|
)
|
|
)
|
|
if metrics
|
|
else ""
|
|
)
|
|
expected: Final = tuple(
|
|
sorted(
|
|
(outcome for outcome in outcomes if outcome.category in ("xfailed", "xpassed")),
|
|
key=lambda outcome: (outcome.category != "xpassed", outcome.nodeid),
|
|
)
|
|
)
|
|
details: Final = (
|
|
"\n".join(
|
|
(
|
|
"",
|
|
"<details>",
|
|
f"<summary>Xfail and xpass reasons ({len(expected)})</summary>",
|
|
"",
|
|
_limited(tuple(_expected_detail(outcome) for outcome in expected)),
|
|
"",
|
|
"</details>",
|
|
)
|
|
)
|
|
if expected
|
|
else ""
|
|
)
|
|
return (
|
|
"\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part)
|
|
+ "\n"
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
parser: Final = argparse.ArgumentParser()
|
|
_ = parser.add_argument("--title", required=True)
|
|
_ = parser.add_argument("report", type=Path)
|
|
arguments: Final = parser.parse_args()
|
|
path: Final = cast(Path, arguments.report)
|
|
title: Final = cast(str, arguments.title)
|
|
try:
|
|
data: Final = _object(cast(object, json.loads(path.read_text(encoding="utf-8"))))
|
|
if data.get("version") != 1:
|
|
sys.stdout.write("Unsupported report version; no report was produced\n")
|
|
return
|
|
sys.stdout.write(render(data, title))
|
|
except (OSError, ValueError, KeyError, TypeError, RecursionError):
|
|
sys.stdout.write("No report was produced (missing or unreadable report)\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|