From 493225e0563a977f3ccd4f8c5ad49206ef912365 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 19 Oct 2026 18:19:05 +0000
Subject: [PATCH 1/5] ci: add non-gating pytest outcome reports
---
.github/actions/pytest-report/action.yml | 38 +++
.github/scripts/render_pytest_report.py | 177 ++++++++++++++
.github/workflows/test-linting.yml | 3 +
.github/workflows/test-unit.yml | 10 +-
scripts/pytest_ci_report.py | 115 +++++++++
.../test_pytest_ci_report.py | 221 ++++++++++++++++++
6 files changed, 563 insertions(+), 1 deletion(-)
create mode 100644 .github/actions/pytest-report/action.yml
create mode 100644 .github/scripts/render_pytest_report.py
create mode 100644 scripts/pytest_ci_report.py
create mode 100644 tests/code_coverage_tests/test_pytest_ci_report.py
diff --git a/.github/actions/pytest-report/action.yml b/.github/actions/pytest-report/action.yml
new file mode 100644
index 00000000000..4c5c342cec3
--- /dev/null
+++ b/.github/actions/pytest-report/action.yml
@@ -0,0 +1,38 @@
+name: "Report pytest outcomes"
+description: >-
+ Append non-gating pytest outcomes and metrics to the job summary, and optionally
+ retain the raw report for later comparison.
+
+inputs:
+ report:
+ description: "Path to the pytest report JSON"
+ required: true
+ title:
+ description: "Heading shown in the job summary"
+ required: true
+ artifact-name:
+ description: "Optional artifact name for retaining the raw report"
+ required: false
+ default: ""
+
+runs:
+ using: composite
+ steps:
+ - name: Render pytest report
+ if: always()
+ continue-on-error: true
+ shell: bash
+ env:
+ REPORT: ${{ inputs.report }}
+ TITLE: ${{ inputs.title }}
+ run: python3 "$GITHUB_ACTION_PATH/../../scripts/render_pytest_report.py" --title "$TITLE" "$REPORT" >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload pytest report
+ if: ${{ always() && inputs.artifact-name != '' }}
+ continue-on-error: true
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: ${{ inputs.report }}
+ retention-days: 7
+ if-no-files-found: ignore
diff --git a/.github/scripts/render_pytest_report.py b/.github/scripts/render_pytest_report.py
new file mode 100644
index 00000000000..7da11445643
--- /dev/null
+++ b/.github/scripts/render_pytest_report.py
@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import sys
+from collections import Counter
+from dataclasses import dataclass
+from itertools import accumulate, groupby
+from pathlib import Path
+from typing import Final, cast
+
+TOTAL_ORDER: Final = ("failed", "passed", "skipped", "xfailed", "xpassed", "error")
+COLUMNS: Final = ("passed", "failed", "error", "skipped", "xfailed", "xpassed")
+
+
+@dataclass(frozen=True, slots=True)
+class Outcome:
+ nodeid: str
+ category: str
+ reason: str
+
+
+def _object(value: object) -> dict[str, object]:
+ if not isinstance(value, dict):
+ raise ValueError("expected an object")
+ return cast(dict[str, object], value)
+
+
+def _array(value: object) -> tuple[dict[str, object], ...]:
+ if not isinstance(value, list):
+ raise ValueError("expected an array")
+ return tuple(_object(row) for row in cast(list[object], value))
+
+
+def _string(value: object) -> str:
+ if not isinstance(value, str):
+ raise ValueError("expected a string")
+ return value
+
+
+def _outcome(row: dict[str, object]) -> Outcome:
+ return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"]))
+
+
+def _cell(value: str | float) -> str:
+ raw: Final = str(value)
+ clipped: Final = raw[:240] + ("..." if len(raw) > 240 else "")
+ return (
+ html.escape(" ".join(clipped.split()), quote=True)
+ .replace("|", "|")
+ .replace("`", "`")
+ .replace("[", "[")
+ .replace("]", "]")
+ .replace("*", "*")
+ .replace("_", "_")
+ .replace("\\", "\")
+ )
+
+
+def _limited(rows: tuple[str, ...]) -> str:
+ candidates: Final = rows[:500]
+ sizes: Final = accumulate(len(row.encode("utf-8")) + 1 for row in candidates)
+ visible: Final = tuple(row for row, size in zip(candidates, sizes) if size <= 200_000)
+ remainder: Final = len(rows) - len(visible)
+ return "\n".join(visible + ((f"\nand {remainder} more",) if remainder else ()))
+
+
+def _file_row(path: str, outcomes: tuple[Outcome, ...]) -> str:
+ counts: Final = Counter(outcome.category for outcome in outcomes)
+ return "| " + _cell(path) + " | " + " | ".join(str(counts[column]) for column in COLUMNS) + " |"
+
+
+def _metric_row(row: dict[str, object]) -> str:
+ value: Final = row["value"]
+ if not isinstance(value, (str, int, float)):
+ raise ValueError("invalid metric value")
+ return (
+ "| "
+ + " | ".join(
+ _cell(value)
+ for value in (
+ _string(row["name"]),
+ value,
+ _string(row["unit"]),
+ _string(row["nodeid"]),
+ )
+ )
+ + " |"
+ )
+
+
+def render(data: dict[str, object], title: str) -> str:
+ outcomes: Final = tuple(_outcome(row) for row in _array(data["outcomes"]))
+ metrics: Final = _array(data["metrics"])
+ counts: Final = Counter(outcome.category for outcome in outcomes)
+ extras: Final = tuple(sorted(set(counts) - set(TOTAL_ORDER)))
+ totals: Final = (
+ ", ".join(
+ f"{counts[category]} {'errors' if category == 'error' and counts[category] != 1 else category}"
+ for category in TOTAL_ORDER + extras
+ if counts[category]
+ )
+ or "No test outcomes recorded"
+ )
+ files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed")
+ grouped: Final = groupby(
+ sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0]
+ )
+ file_rows: Final = tuple(_file_row(path, tuple(group)) for path, group in grouped if path in files)
+ table: Final = "\n".join(
+ (
+ "| File | passed | failed | error | skipped | xfailed | xpassed |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+ _limited(file_rows),
+ )
+ )
+ metric_table: Final = (
+ "\n".join(
+ (
+ "",
+ "| Metric | Value | Unit | Test |",
+ "| --- | --- | --- | --- |",
+ _limited(tuple(_metric_row(row) for row in metrics)),
+ )
+ )
+ if metrics
+ else ""
+ )
+ expected: Final = tuple(
+ sorted(
+ (outcome for outcome in outcomes if outcome.category in ("xfailed", "xpassed")),
+ key=lambda outcome: (outcome.category != "xpassed", outcome.nodeid),
+ )
+ )
+ details: Final = (
+ "\n".join(
+ (
+ "",
+ "",
+ "Xfail and xpass reasons
",
+ "",
+ _limited(
+ tuple(
+ f"- {outcome.category}: {_cell(outcome.nodeid)}: {_cell(outcome.reason)}"
+ for outcome in expected
+ )
+ ),
+ "",
+ " ",
+ )
+ )
+ if expected
+ else ""
+ )
+ return "\n".join((f"## {_cell(title)}", "", totals, "", table, metric_table, details, ""))
+
+
+def main() -> None:
+ parser: Final = argparse.ArgumentParser()
+ _ = parser.add_argument("--title", required=True)
+ _ = parser.add_argument("report", type=Path)
+ arguments: Final = parser.parse_args()
+ path: Final = cast(Path, arguments.report)
+ title: Final = cast(str, arguments.title)
+ try:
+ data: Final = _object(cast(object, json.loads(path.read_text(encoding="utf-8"))))
+ if data.get("version") != 1:
+ sys.stdout.write("Unsupported report version; no report was produced\n")
+ return
+ sys.stdout.write(render(data, title))
+ except (OSError, ValueError, KeyError, TypeError, RecursionError):
+ sys.stdout.write("No report was produced (missing or unreadable report)\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index 3b9a2614725..0e8e9fb8ef9 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -549,6 +549,9 @@ jobs:
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
+ - name: test_pytest_ci_report
+ run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_pytest_ci_report.py
+
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index cb1b595f3f0..bd576b002b7 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -1048,7 +1048,7 @@ jobs:
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
fi
set +e
- uv run --no-sync pytest "${pytest_args[@]}" \
+ uv run --no-sync pytest -p pytest_ci_report --ci-report=pytest-report.json "${pytest_args[@]}" \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
"${xdist_args[@]}" \
@@ -1078,6 +1078,14 @@ jobs:
name: coverage-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
+
+ - name: Report test outcomes
+ if: always() && steps.changes.outputs.decision != 'skip'
+ uses: ./.github/actions/pytest-report
+ with:
+ report: pytest-report.json
+ title: ${{ matrix.shard }}
+ artifact-name: pytest-report-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
ui-unit:
name: ui-unit
permissions:
diff --git a/scripts/pytest_ci_report.py b/scripts/pytest_ci_report.py
new file mode 100644
index 00000000000..ecbec5c3165
--- /dev/null
+++ b/scripts/pytest_ci_report.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Final, Protocol, cast
+
+import pytest
+
+
+class CIMetric(Protocol):
+ def __call__(self, name: str, value: str | float, unit: str = "") -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Outcome:
+ nodeid: str
+ when: str
+ category: str
+ reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class Metric:
+ nodeid: str
+ name: str
+ value: str | float
+ unit: str
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+ outcomes: tuple[Outcome, ...] = ()
+ metrics: tuple[Metric, ...] = ()
+ version: int = 1
+
+
+REPORT: Final = pytest.StashKey[Report]()
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+ parser.addoption("--ci-report", default=None, help="Write non-gating CI outcomes and metrics as JSON")
+
+
+def pytest_configure(config: pytest.Config) -> None:
+ path: Final[object] = config.getoption("ci_report")
+ if not isinstance(path, str) or hasattr(config, "workerinput"):
+ return
+ config.stash[REPORT] = Report()
+ config.pluginmanager.register(CIReporter(config, Path(path)), "ci-report-writer")
+
+
+def _reason(report: pytest.TestReport) -> str:
+ xfail: Final[object] = getattr(report, "wasxfail", None)
+ if isinstance(xfail, str):
+ return xfail
+ if report.skipped and isinstance(report.longrepr, tuple):
+ return str(report.longrepr[2])
+ return ""
+
+
+def _metric(nodeid: str, prop: object) -> Metric | None:
+ if not isinstance(prop, (tuple, list)) or len(prop) != 2 or prop[0] != "ci_metric":
+ return None
+ value: Final[object] = prop[1]
+ if not isinstance(value, (tuple, list)) or len(value) != 3:
+ return None
+ name, amount, unit = cast(tuple[object, object, object], value)
+ if isinstance(name, str) and isinstance(amount, (str, int, float)) and isinstance(unit, str):
+ return Metric(nodeid, name, amount, unit)
+ return None
+
+
+@dataclass(frozen=True, slots=True)
+class CIReporter:
+ config: pytest.Config
+ path: Path
+
+ def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+ status: Final = cast(
+ tuple[str, str, object],
+ self.config.hook.pytest_report_teststatus(
+ report=report,
+ config=self.config,
+ ),
+ )
+ category: Final = status[0]
+ previous: Final = self.config.stash[REPORT]
+ outcomes: Final = previous.outcomes + (
+ (Outcome(report.nodeid, report.when, category, _reason(report)),) if category not in ("", "rerun") else ()
+ )
+ metrics: Final = (
+ tuple(metric for metric in previous.metrics if metric.nodeid != report.nodeid)
+ + tuple(metric for prop in report.user_properties if (metric := _metric(report.nodeid, prop)) is not None)
+ if report.when == "teardown"
+ else previous.metrics
+ )
+ self.config.stash[REPORT] = Report(outcomes, metrics)
+
+ def pytest_sessionfinish(self) -> None:
+ try:
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ _ = self.path.write_text(json.dumps(asdict(self.config.stash[REPORT])), encoding="utf-8")
+ except (OSError, ValueError) as error:
+ sys.stderr.write(f"Could not write CI report: {error}\n")
+
+
+@pytest.fixture
+def ci_metric(request: pytest.FixtureRequest) -> CIMetric:
+ def record(name: str, value: str | float, unit: str = "") -> None:
+ if request.config.getoption("ci_report") is not None:
+ request.node.user_properties.append(("ci_metric", (name, value, unit)))
+
+ return record
diff --git a/tests/code_coverage_tests/test_pytest_ci_report.py b/tests/code_coverage_tests/test_pytest_ci_report.py
new file mode 100644
index 00000000000..9cb0e316429
--- /dev/null
+++ b/tests/code_coverage_tests/test_pytest_ci_report.py
@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import html
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
+
+import pytest
+from pydantic import BaseModel, ConfigDict, TypeAdapter
+
+ROOT: Final = Path(__file__).resolve().parents[2]
+RENDERER: Final = ROOT / ".github/scripts/render_pytest_report.py"
+SUITE: Final = """
+from pathlib import Path
+import pytest
+
+def test_pass(ci_metric):
+ ci_metric("Size", 41.2, "MB")
+
+def test_fail():
+ assert False, "permanent failure"
+
+def test_skip():
+ pytest.skip("not available")
+
+@pytest.mark.xfail(reason="known bug")
+def test_xfail():
+ assert False
+
+@pytest.mark.xfail(reason="fixed bug")
+def test_xpass():
+ pass
+
+def test_retry():
+ marker = Path(__file__).with_name("attempted")
+ first = not marker.exists()
+ marker.touch()
+ assert not first, "retry me"
+
+@pytest.fixture
+def teardown_error():
+ yield
+ raise RuntimeError("cleanup failed")
+
+def test_teardown(teardown_error):
+ pass
+"""
+
+
+class ReportOutcome(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ nodeid: str
+ when: str
+ category: str
+ reason: str
+
+
+class ReportMetric(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ nodeid: str
+ name: str
+ value: str | float
+ unit: str
+
+
+class ReportData(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ version: int
+ outcomes: tuple[ReportOutcome, ...]
+ metrics: tuple[ReportMetric, ...]
+
+
+REPORT_DATA: Final = TypeAdapter(ReportData)
+
+
+def run_suite(directory: Path, source: str = SUITE, *args: str) -> subprocess.CompletedProcess[str]:
+ directory.mkdir(exist_ok=True)
+ _ = (directory / "test_sample.py").write_text(source)
+ return subprocess.run(
+ (
+ sys.executable,
+ "-P",
+ "-m",
+ "pytest",
+ "-c",
+ "/dev/null",
+ f"--rootdir={directory}",
+ "--noconftest",
+ "-p",
+ "pytest_ci_report",
+ "-p",
+ "no:cacheprovider",
+ "--color=no",
+ "-rA",
+ *args,
+ "test_sample.py",
+ ),
+ cwd=directory,
+ env={**os.environ, "PYTHONPATH": str(ROOT / "scripts"), "PYTEST_ADDOPTS": ""},
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=90,
+ )
+
+
+def render(path: Path) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ (sys.executable, "-P", str(RENDERER), "--title", "CI outcomes", str(path)),
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=10,
+ )
+
+
+@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist"))
+def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, workers: tuple[str, ...]) -> None:
+ report: Final = tmp_path / "report.json"
+ result: Final = run_suite(
+ tmp_path, SUITE, "--ci-report", str(report), "--reruns", "1", "--only-rerun", "retry me", *workers
+ )
+ assert result.returncode == 1, result.stdout + result.stderr
+ data: Final = REPORT_DATA.validate_json(report.read_text())
+ assert data.version == 1
+ assert sorted((row.nodeid.split("::")[-1], row.when, row.category, row.reason) for row in data.outcomes) == [
+ ("test_fail", "call", "failed", ""),
+ ("test_pass", "call", "passed", ""),
+ ("test_retry", "call", "passed", ""),
+ ("test_skip", "call", "skipped", "Skipped: not available"),
+ ("test_teardown", "call", "passed", ""),
+ ("test_teardown", "teardown", "error", ""),
+ ("test_xfail", "call", "xfailed", "known bug"),
+ ("test_xpass", "call", "xpassed", "fixed bug"),
+ ]
+ assert data.metrics == (ReportMetric(nodeid="test_sample.py::test_pass", name="Size", value=41.2, unit="MB"),)
+ assert tuple(tmp_path.glob("*.json")) == (report,)
+ rendered: Final = render(report)
+ assert rendered.returncode == 0, rendered.stderr
+ terminal: Final = re.findall(
+ r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", result.stdout.splitlines()[-1]
+ )
+ assert ", ".join(terminal) in rendered.stdout
+ assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout)
+ assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug")
+
+
+def test_maxfail_still_writes_report(tmp_path: Path) -> None:
+ result: Final = run_suite(
+ tmp_path, "def test_failure():\n assert False\n", "--ci-report=report.json", "--maxfail=1"
+ )
+ assert result.returncode == 1
+ data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text())
+ assert data.outcomes == (
+ ReportOutcome(nodeid="test_sample.py::test_failure", when="call", category="failed", reason=""),
+ )
+
+
+def test_report_is_opt_in(tmp_path: Path) -> None:
+ result: Final = run_suite(tmp_path, "def test_pass(ci_metric):\n ci_metric('Size', 1)\n")
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert not tuple(tmp_path.glob("*.json"))
+
+
+def test_unwritable_report_does_not_fail_tests(tmp_path: Path) -> None:
+ result: Final = run_suite(tmp_path, "def test_pass():\n pass\n", f"--ci-report={tmp_path}")
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+@pytest.mark.parametrize("content", (None, "{", '{"version": 2}', '{"version": 1, "outcomes": null}'))
+def test_unavailable_or_unknown_reports_are_non_gating(tmp_path: Path, content: str | None) -> None:
+ report: Final = tmp_path / "report.json"
+ if content is not None:
+ _ = report.write_text(content)
+ result: Final = render(report)
+ assert result.returncode == 0, result.stderr
+ assert "No report was produced" in result.stdout or "Unsupported report version" in result.stdout
+
+
+@pytest.mark.parametrize("reason", ("known bug", "|\n" * 10000), ids=("entry-limit", "byte-limit"))
+def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str) -> None:
+ report: Final = tmp_path / "report.json"
+ _ = report.write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "metrics": [],
+ "outcomes": [
+ {"nodeid": f"test_sample.py::test_{index}", "when": "call", "category": "xfailed", "reason": reason}
+ for index in range(600)
+ ],
+ }
+ )
+ )
+ result: Final = render(report)
+ assert result.returncode == 0
+ visible: Final = result.stdout.count("- xfailed:")
+ assert visible <= 500
+ assert f"and {600 - visible} more" in result.stdout
+ assert len(result.stdout.encode()) < 1_000_000
+ assert "" not in result.stdout
+
+
+def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None:
+ result: Final = run_suite(
+ tmp_path,
+ "def test_pass(ci_metric):\n ci_metric('Size', 1, 'MB')\n",
+ "--ci-report=report.json",
+ "--junitxml=report.xml",
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert 'name="ci_metric"' in (tmp_path / "report.xml").read_text()
+ data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text())
+ assert data.metrics[0].value == 1
From 19dedb55495d2872cef15f3a3586e1b521bad311 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 19 Oct 2026 18:33:46 +0000
Subject: [PATCH 2/5] ci: report native wheel and rust extension pytest
outcomes
---
.github/scripts/verify_linux_native_wheel.py | 190 ++++++------------
.github/workflows/test-rust.yml | 32 ++-
Makefile | 4 +-
scripts/pytest_ci_report.py | 21 +-
.../test_pytest_ci_report.py | 63 ++++++
.../test_verify_linux_native_wheel.py | 166 ++++++++++-----
tests/wheel/AGENTS.md | 7 +
tests/wheel/conftest.py | 26 +++
tests/wheel/test_linux_native_wheel.py | 89 ++++++++
9 files changed, 417 insertions(+), 181 deletions(-)
create mode 100644 tests/wheel/AGENTS.md
create mode 100644 tests/wheel/conftest.py
create mode 100644 tests/wheel/test_linux_native_wheel.py
diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py
index 4c4cb399dce..99f282e7bc3 100644
--- a/.github/scripts/verify_linux_native_wheel.py
+++ b/.github/scripts/verify_linux_native_wheel.py
@@ -6,7 +6,8 @@ import re
import subprocess
import sys
import zipfile
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
from itertools import product
from pathlib import Path, PurePosixPath
from types import MappingProxyType, ModuleType
@@ -17,6 +18,36 @@ EXPECTED_ABI_TAG: Final = "abi3"
EXPECTED_PLATFORM_TAG: Final = "linux_x86_64"
+@dataclass(frozen=True, slots=True)
+class WheelInspectionError:
+ message: str
+
+
+@dataclass(frozen=True, slots=True)
+class WheelFacts:
+ python_tag: str
+ abi_tag: str
+ platform_tag: str
+ expected_dist_info_directory: str
+ dist_info_directories: frozenset[str]
+ dist_info_file_counts: Mapping[str, int]
+ wheel_metadata_tags: tuple[str, ...]
+ expanded_filename_tags: frozenset[str]
+ compressed_size: int
+ uncompressed_size: int
+ native_size: int
+ debug_sections: tuple[str, ...]
+ static_symbol_table_absent: bool
+ extension_entry_point_present: bool
+ native_module_loads: bool
+ panic_test_hook_absent: bool
+ duplicated_vocabularies: tuple[str, ...]
+ unexpected_members: tuple[str, ...]
+ commit_sha: str
+ rustc_version: str
+ maturin_version: str
+
+
class CommandRunner(Protocol):
def __call__(
self,
@@ -65,31 +96,22 @@ def _load_native_module(native_path: Path) -> ModuleType | None:
return native_module
-def main(
- argv: Sequence[str] | None = None,
+def inspect_wheel(
+ wheel: Path,
environment: Mapping[str, str] | None = None,
load_native_module: Callable[[Path], ModuleType | None] = _load_native_module,
run_command: CommandRunner = _run_command,
-) -> int:
- arguments: Final = tuple(sys.argv if argv is None else argv)
+) -> WheelFacts | WheelInspectionError:
resolved_environment: Final = os.environ if environment is None else environment
- if len(arguments) != 2:
- sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n")
- return 2
-
- wheel: Final = Path(arguments[1])
wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3)
if len(wheel_tags) != 4:
- sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n")
- return 1
+ return WheelInspectionError(f"cannot parse wheel tags from {wheel.name}")
wheel_identity: Final = wheel_tags[0].split("-")
if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]:
- sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n")
- return 1
+ return WheelInspectionError(f"unexpected wheel identity: {wheel_tags[0]}")
expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info"
- expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,))
python_tag: Final = wheel_tags[1]
abi_tag: Final = wheel_tags[2]
platform_tag: Final = wheel_tags[3]
@@ -121,8 +143,7 @@ def main(
if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
)
if len(native_members) != 1:
- sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
- return 1
+ return WheelInspectionError(f"expected one native extension, found {len(native_members)}")
unexpected_members: Final = tuple(
member.filename
@@ -145,10 +166,6 @@ def main(
and archive.read(member) in native_bytes
)
- wheel_metadata_tags_match: Final = (
- len(wheel_metadata_tags) == len(expanded_filename_tags)
- and frozenset(wheel_metadata_tags) == expanded_filename_tags
- )
commit_sha: Final = resolved_environment.get(
"RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown")
)
@@ -161,39 +178,9 @@ def main(
pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text()
maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject)
if maturin_match is None:
- sys.stderr.write("build-system does not pin an exact Maturin version\n")
- return 1
+ return WheelInspectionError("build-system does not pin an exact Maturin version")
maturin_version: Final = maturin_match.group(1)
- native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100
- size_report: Final = "\n".join(
- (
- "## Native wheel build report",
- "",
- "| Build | Value |",
- "| --- | --- |",
- f"| Commit | `{commit_sha}` |",
- f"| Platform | `{platform_tag}` |",
- f"| Python ABI | `{python_tag}-{abi_tag}` |",
- f"| Rust compiler | `{rustc_version}` |",
- f"| Maturin | `{maturin_version}` |",
- "| Cargo profile | `release` |",
- "",
- "| Artifact | Size |",
- "| --- | ---: |",
- f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |",
- f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |",
- f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |",
- f"| Native share | {native_percentage:.2f}% |",
- "",
- )
- )
- summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY")
- if summary_path is None:
- sys.stdout.write(size_report)
- else:
- Path(summary_path).write_text(size_report)
-
sections: Final = run_command(
("readelf", "--sections", "--wide", str(native_path)),
check=True,
@@ -201,7 +188,6 @@ def main(
text=True,
).stdout
debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections)
- debug_sections_absent: Final = not debug_sections
static_symbol_table_absent: Final = ".symtab" not in sections
dynamic_symbols: Final = run_command(
@@ -214,80 +200,26 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
- native_size_limit: Final = 48_000_000
- native_size_within_limit: Final = native_member.file_size <= native_size_limit
- validations: Final = (
- (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
- (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG),
- (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG),
- ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories),
- (
- "Required dist-info files are present exactly once",
- all(count == 1 for count in dist_info_file_counts.values()),
- ),
- ("Wheel metadata tags match the filename", wheel_metadata_tags_match),
- ("Debug sections are absent", debug_sections_absent),
- ("Static symbol table is absent", static_symbol_table_absent),
- ("Python extension entry point is present", extension_entry_point_present),
- ("Native module loads", native_module_loads),
- ("Production module omits the panic test hook", panic_test_hook_absent),
- (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
- ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
- ("Wheel contents are valid", not unexpected_members),
+ return WheelFacts(
+ python_tag=python_tag,
+ abi_tag=abi_tag,
+ platform_tag=platform_tag,
+ expected_dist_info_directory=expected_dist_info_directory,
+ dist_info_directories=dist_info_directories,
+ dist_info_file_counts=dist_info_file_counts,
+ wheel_metadata_tags=wheel_metadata_tags,
+ expanded_filename_tags=expanded_filename_tags,
+ compressed_size=wheel.stat().st_size,
+ uncompressed_size=uncompressed_wheel_size,
+ native_size=native_member.file_size,
+ debug_sections=debug_sections,
+ static_symbol_table_absent=static_symbol_table_absent,
+ extension_entry_point_present=extension_entry_point_present,
+ native_module_loads=native_module_loads,
+ panic_test_hook_absent=panic_test_hook_absent,
+ duplicated_vocabularies=duplicated_vocabularies,
+ unexpected_members=unexpected_members,
+ commit_sha=commit_sha,
+ rustc_version=rustc_version,
+ maturin_version=maturin_version,
)
-
- verified_report: Final = size_report + "\n".join(
- ("", "| Validation | Expected | Result |", "| --- | --- | :---: |")
- + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations)
- + ("",)
- )
- if summary_path is not None:
- Path(summary_path).write_text(verified_report)
-
- invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values())
- validation_errors: Final = tuple(
- message
- for failed, message in (
- (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"),
- (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"),
- (not extension_entry_point_present, "native extension does not export PyInit__native"),
- (
- python_tag != EXPECTED_PYTHON_TAG,
- f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}",
- ),
- (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"),
- (
- platform_tag != EXPECTED_PLATFORM_TAG,
- f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
- ),
- (
- dist_info_directories != expected_dist_info_directories,
- f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
- f"found {', '.join(sorted(dist_info_directories))}",
- ),
- (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"),
- (
- not invalid_dist_info_files and not wheel_metadata_tags_match,
- f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
- f"found {', '.join(sorted(wheel_metadata_tags))}",
- ),
- (
- native_module is not None and not panic_test_hook_absent,
- "production native module exposes _panic_for_test",
- ),
- (
- not native_size_within_limit,
- f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
- f"{native_member.file_size / 1_000_000:.2f} MB",
- ),
- (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
- )
- if failed
- )
- sys.stderr.write("".join(f"{message}\n" for message in validation_errors))
-
- return 0 if all(passed for _, passed in validations) else 1
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml
index 636a5031e5f..47035cabf8a 100644
--- a/.github/workflows/test-rust.yml
+++ b/.github/workflows/test-rust.yml
@@ -30,6 +30,10 @@ on:
- ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
+ - ".github/actions/pytest-report/**"
+ - ".github/scripts/render_pytest_report.py"
+ - "scripts/pytest_ci_report.py"
+ - "tests/wheel/**"
- "tests/unit/rust_bridge/native_route_wheel_test.py"
- ".github/workflows/test-rust.yml"
pull_request:
@@ -64,6 +68,10 @@ on:
- ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
+ - ".github/actions/pytest-report/**"
+ - ".github/scripts/render_pytest_report.py"
+ - "scripts/pytest_ci_report.py"
+ - "tests/wheel/**"
- "tests/unit/rust_bridge/native_route_wheel_test.py"
- ".github/workflows/test-rust.yml"
@@ -189,14 +197,34 @@ jobs:
- run: uv build --wheel --out-dir dist
- - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
+ - name: Check native wheel
+ run: |
+ set -- dist/*.whl
+ [ "$#" -eq 1 ]
+ uv run --no-project --with pytest==9.0.3 python -m pytest tests/wheel --wheel "$1" -p pytest_ci_report --ci-report=wheel-report.json
env:
RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ - name: Report native wheel outcomes
+ if: always()
+ uses: ./.github/actions/pytest-report
+ with:
+ report: wheel-report.json
+ title: Native wheel
+ artifact-name: wheel-report-${{ github.run_id }}-${{ github.run_attempt }}
+
- run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
- name: Run pytest tests/test_litellm_rust with the compiled extension
- run: make test-rust-extension
+ run: make test-rust-extension PYTEST_REPORT_ARGS="-p pytest_ci_report --ci-report=rust-extension-report.json"
+
+ - name: Report Rust extension outcomes
+ if: always()
+ uses: ./.github/actions/pytest-report
+ with:
+ report: rust-extension-report.json
+ title: Rust extension
+ artifact-name: rust-extension-report-${{ github.run_id }}-${{ github.run_attempt }}
- run: >-
uv build --wheel --out-dir panic-dist
diff --git a/Makefile b/Makefile
index a7a12a8635a..539dbc28c92 100644
--- a/Makefile
+++ b/Makefile
@@ -279,6 +279,8 @@ pre-commit:
@$(MAKE) check
# Testing targets
+PYTEST_REPORT_ARGS ?=
+
test-rust-extension:
@temporary=$$(mktemp -d) && \
trap 'rm -rf "$$temporary"' EXIT HUP INT TERM && \
@@ -291,7 +293,7 @@ test-rust-extension:
--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
litellm.rust_bridge._native && \
LITELLM_LOCAL_MODEL_COST_MAP=True \
- "$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
+ "$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension $(PYTEST_REPORT_ARGS) tests/test_litellm_rust
rust-sqlx-prepare:
cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
diff --git a/scripts/pytest_ci_report.py b/scripts/pytest_ci_report.py
index ecbec5c3165..f63de455f8f 100644
--- a/scripts/pytest_ci_report.py
+++ b/scripts/pytest_ci_report.py
@@ -51,12 +51,14 @@ def pytest_configure(config: pytest.Config) -> None:
config.pluginmanager.register(CIReporter(config, Path(path)), "ci-report-writer")
-def _reason(report: pytest.TestReport) -> str:
+def _reason(report: pytest.TestReport | pytest.CollectReport) -> str:
xfail: Final[object] = getattr(report, "wasxfail", None)
if isinstance(xfail, str):
return xfail
if report.skipped and isinstance(report.longrepr, tuple):
return str(report.longrepr[2])
+ if isinstance(report, pytest.CollectReport) and report.failed:
+ return str(report.longrepr)
return ""
@@ -77,6 +79,23 @@ class CIReporter:
config: pytest.Config
path: Path
+ def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+ status: Final = cast(
+ tuple[str, str, object],
+ self.config.hook.pytest_report_teststatus(
+ report=report,
+ config=self.config,
+ ),
+ )
+ category: Final = status[0]
+ if category not in ("error", "skipped"):
+ return
+ previous: Final = self.config.stash[REPORT]
+ self.config.stash[REPORT] = Report(
+ previous.outcomes + (Outcome(report.nodeid, "collect", category, _reason(report)),),
+ previous.metrics,
+ )
+
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
status: Final = cast(
tuple[str, str, object],
diff --git a/tests/code_coverage_tests/test_pytest_ci_report.py b/tests/code_coverage_tests/test_pytest_ci_report.py
index 9cb0e316429..fdd72478998 100644
--- a/tests/code_coverage_tests/test_pytest_ci_report.py
+++ b/tests/code_coverage_tests/test_pytest_ci_report.py
@@ -152,6 +152,69 @@ def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, worke
assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug")
+@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist"))
+def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, workers: tuple[str, ...]) -> None:
+ report: Final = tmp_path / "report.json"
+ result: Final = run_suite(
+ tmp_path,
+ "import missing_ci_test_module\n",
+ "--ci-report",
+ str(report),
+ *workers,
+ )
+ assert result.returncode != 0, result.stdout + result.stderr
+ data: Final = REPORT_DATA.validate_json(report.read_text())
+ assert len(data.outcomes) == 1
+ outcome: Final = data.outcomes[0]
+ assert outcome.nodeid == "test_sample.py"
+ assert outcome.when == "collect"
+ assert outcome.category == "error"
+ assert "missing_ci_test_module" in outcome.reason
+ terminal_summary: Final = next(
+ line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line)
+ )
+ terminal_counts: Final = re.findall(
+ r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary
+ )
+ assert terminal_counts == ["1 error"]
+ rendered: Final = render(report)
+ assert rendered.returncode == 0, rendered.stderr
+ assert ", ".join(terminal_counts) in rendered.stdout
+
+
+def test_collection_skips_are_reported(tmp_path: Path) -> None:
+ report: Final = tmp_path / "report.json"
+ result: Final = run_suite(
+ tmp_path,
+ "import pytest\npytest.skip('not available', allow_module_level=True)\n",
+ "--ci-report",
+ str(report),
+ )
+ data: Final = REPORT_DATA.validate_json(report.read_text())
+ assert data.outcomes == (
+ ReportOutcome(nodeid="test_sample.py", when="collect", category="skipped", reason="Skipped: not available"),
+ )
+ assert "1 skipped" in result.stdout
+ rendered: Final = render(report)
+ assert rendered.returncode == 0, rendered.stderr
+ assert "1 skipped" in rendered.stdout
+
+
+def test_plugin_loads_with_repository_config_without_pythonpath() -> None:
+ environment: Final = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
+ result: Final = subprocess.run(
+ (sys.executable, "-P", "-m", "pytest", "-p", "pytest_ci_report", "--help"),
+ cwd=ROOT,
+ env=environment,
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=90,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert "--ci-report" in result.stdout
+
+
def test_maxfail_still_writes_report(tmp_path: Path) -> None:
result: Final = run_suite(
tmp_path, "def test_failure():\n assert False\n", "--ci-report=report.json", "--maxfail=1"
diff --git a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py
index 0eae041f535..9ae7b199e61 100644
--- a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py
+++ b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py
@@ -4,7 +4,7 @@ import importlib.util
import subprocess
import sys
import zipfile
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Mapping
from pathlib import Path
from types import MappingProxyType, ModuleType
from typing import Final, Protocol, cast
@@ -23,16 +23,35 @@ class _CommandRunner(Protocol):
) -> subprocess.CompletedProcess[str]: ...
+class _WheelFacts(Protocol):
+ python_tag: str
+ abi_tag: str
+ platform_tag: str
+ expected_dist_info_directory: str
+ dist_info_directories: frozenset[str]
+ dist_info_file_counts: Mapping[str, int]
+ wheel_metadata_tags: tuple[str, ...]
+ expanded_filename_tags: frozenset[str]
+ compressed_size: int
+ uncompressed_size: int
+ native_size: int
+ debug_sections: tuple[str, ...]
+ static_symbol_table_absent: bool
+ extension_entry_point_present: bool
+ native_module_loads: bool
+ panic_test_hook_absent: bool
+ duplicated_vocabularies: tuple[str, ...]
+ unexpected_members: tuple[str, ...]
+
+
class _VerifierModule(Protocol):
- main: Callable[
- [
- Sequence[str] | None,
- Mapping[str, str] | None,
- Callable[[Path], ModuleType | None],
- _CommandRunner,
- ],
- int,
- ]
+ def inspect_wheel(
+ self,
+ wheel: Path,
+ environment: Mapping[str, str] | None = None,
+ load_native_module: Callable[[Path], ModuleType | None] = ...,
+ run_command: _CommandRunner = ...,
+ ) -> _WheelFacts: ...
_REPO_ROOT: Final = Path(__file__).resolve().parents[3]
@@ -80,21 +99,28 @@ def _write_wheel(
return wheel
-def _fake_subprocess_run(
- command: tuple[str, ...],
- *,
- check: bool,
- capture_output: bool,
- text: bool,
-) -> subprocess.CompletedProcess[str]:
- assert check and capture_output and text
- if command == ("rustc", "--version"):
- return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="")
- if "--sections" in command:
- return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="")
- if "--dyn-syms" in command:
- return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="")
- raise AssertionError(f"unexpected subprocess command: {command}")
+_STRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n"
+_UNSTRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n[ 2] .debug_info PROGBITS\n[ 3] .symtab SYMTAB\n"
+
+
+def _fake_runner(sections: str = _STRIPPED_SECTIONS) -> _CommandRunner:
+ def run(
+ command: tuple[str, ...],
+ *,
+ check: bool,
+ capture_output: bool,
+ text: bool,
+ ) -> subprocess.CompletedProcess[str]:
+ assert check and capture_output and text
+ if command == ("rustc", "--version"):
+ return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="")
+ if "--sections" in command:
+ return subprocess.CompletedProcess(command, 0, stdout=sections, stderr="")
+ if "--dyn-syms" in command:
+ return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="")
+ raise AssertionError(f"unexpected subprocess command: {command}")
+
+ return run
class _NativeModuleWithPanicHook(ModuleType):
@@ -102,11 +128,12 @@ class _NativeModuleWithPanicHook(ModuleType):
return None
-def _run_verifier(
+def _inspect(
wheel: Path,
*,
exposes_panic: bool = False,
-) -> int:
+ sections: str = _STRIPPED_SECTIONS,
+) -> _WheelFacts:
native_module: Final = (
_NativeModuleWithPanicHook("litellm.rust_bridge._native")
if exposes_panic
@@ -116,33 +143,61 @@ def _run_verifier(
def _fake_load_native_module(_: Path) -> ModuleType:
return native_module
- environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")})
- return verifier.main(
- (str(_MODULE_PATH), str(wheel)),
+ environment: Final = MappingProxyType({"RELEASE_WHEEL_COMMIT_SHA": "0" * 40})
+ return verifier.inspect_wheel(
+ wheel,
environment,
_fake_load_native_module,
- _fake_subprocess_run,
+ _fake_runner(sections),
)
-def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None:
+def test_reports_expected_release_wheel_tags(tmp_path: Path) -> None:
+ native_bytes: Final = b"synthetic native extension"
+ wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, native_bytes=native_bytes)
+
+ facts: Final = _inspect(wheel)
+
+ assert (facts.python_tag, facts.abi_tag, facts.platform_tag) == ("cp310", "abi3", "linux_x86_64")
+ assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,))
+ assert facts.expected_dist_info_directory == _DIST_INFO
+ assert dict(facts.dist_info_file_counts) == {"METADATA": 1, "RECORD": 1, "WHEEL": 1}
+ assert frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags
+ assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags)
+ assert facts.native_size == len(native_bytes)
+ assert facts.compressed_size == wheel.stat().st_size
+ assert facts.uncompressed_size >= facts.native_size > 0
+ assert not facts.duplicated_vocabularies
+ assert not facts.unexpected_members
+ assert facts.extension_entry_point_present and facts.native_module_loads
+ assert facts.panic_test_hook_absent
+
+
+def test_reports_debug_sections_and_static_symbol_table(tmp_path: Path) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
- assert _run_verifier(wheel) == 0
+ facts: Final = _inspect(wheel, sections=_UNSTRIPPED_SECTIONS)
+
+ assert facts.debug_sections == (".debug_",)
+ assert not facts.static_symbol_table_absent
-def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None:
+def test_reports_cp312_version_specific_tags(tmp_path: Path) -> None:
tag: Final = "cp312-cp312-linux_x86_64"
wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,))
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert (facts.python_tag, facts.abi_tag) == ("cp312", "cp312")
-def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None:
+def test_reports_non_linux_platform_tag(tmp_path: Path) -> None:
tag: Final = "cp310-abi3-win_amd64"
wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,))
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert facts.platform_tag == "win_amd64"
@pytest.mark.parametrize(
@@ -150,16 +205,18 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None:
(None, ("cp312-cp312-linux_x86_64",)),
ids=("missing", "mismatched"),
)
-def test_rejects_missing_or_mismatched_wheel_metadata_tag(
+def test_reports_missing_or_mismatched_wheel_metadata_tag(
tmp_path: Path,
metadata_tags: tuple[str, ...] | None,
) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags)
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert frozenset(facts.wheel_metadata_tags) != facts.expanded_filename_tags
-def test_rejects_wheel_metadata_from_wrong_dist_info_directory(
+def test_reports_wheel_metadata_from_wrong_dist_info_directory(
tmp_path: Path,
) -> None:
wheel: Final = _write_wheel(
@@ -168,20 +225,27 @@ def test_rejects_wheel_metadata_from_wrong_dist_info_directory(
dist_info="decoy-1.0.0.dist-info",
)
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert facts.dist_info_directories == frozenset(("decoy-1.0.0.dist-info",))
+ assert facts.dist_info_directories != frozenset((facts.expected_dist_info_directory,))
+ assert dict(facts.dist_info_file_counts) == {"METADATA": 0, "RECORD": 0, "WHEEL": 0}
-def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None:
+def test_reports_duplicate_wheel_metadata_tags(tmp_path: Path) -> None:
wheel: Final = _write_wheel(
tmp_path,
filename_tag=_EXPECTED_TAG,
metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG),
)
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert facts.wheel_metadata_tags == (_EXPECTED_TAG, _EXPECTED_TAG)
+ assert len(facts.wheel_metadata_tags) != len(facts.expanded_filename_tags)
-def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
+def test_reports_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
with pytest.warns(UserWarning, match="Duplicate name"):
wheel: Final = _write_wheel(
tmp_path,
@@ -189,13 +253,17 @@ def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
duplicate_wheel=True,
)
- assert _run_verifier(wheel) == 1
+ facts: Final = _inspect(wheel)
+
+ assert facts.dist_info_file_counts["WHEEL"] == 2
-def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None:
+def test_reports_production_module_exposing_panic_hook(tmp_path: Path) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
- assert _run_verifier(wheel, exposes_panic=True) == 1
+ facts: Final = _inspect(wheel, exposes_panic=True)
+
+ assert not facts.panic_test_hook_absent
@pytest.mark.parametrize("embedded", (False, True))
@@ -209,4 +277,6 @@ def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None:
with zipfile.ZipFile(wheel, "a") as archive:
archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks)
- assert _run_verifier(wheel) == (1 if embedded else 0)
+ facts: Final = _inspect(wheel)
+
+ assert bool(facts.duplicated_vocabularies) == embedded
diff --git a/tests/wheel/AGENTS.md b/tests/wheel/AGENTS.md
new file mode 100644
index 00000000000..fe8dbb52523
--- /dev/null
+++ b/tests/wheel/AGENTS.md
@@ -0,0 +1,7 @@
+# Release wheel checks
+
+- Run against a release-built wheel passed with `--wheel`
+- Only run in the `rust-wheel` job, never select from a unit shard
+- May invoke `readelf` and load the compiled native extension
+- Keep facts in `.github/scripts/verify_linux_native_wheel.py` and assertions here
+- Record build metrics with `ci_metric`, never write a separate summary
diff --git a/tests/wheel/conftest.py b/tests/wheel/conftest.py
new file mode 100644
index 00000000000..bb31ba69ec2
--- /dev/null
+++ b/tests/wheel/conftest.py
@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".github" / "scripts")) # test-quality-ok: the wheel inspector lives in .github/scripts, outside any importable package
+
+from verify_linux_native_wheel import WheelFacts, WheelInspectionError, inspect_wheel # noqa: E402 # CI helpers live outside Python packages
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+ parser.addoption("--wheel", help="Release-built native wheel to inspect")
+
+
+@pytest.fixture(scope="session")
+def facts(request: pytest.FixtureRequest) -> WheelFacts:
+ wheel: Final[object] = request.config.getoption("wheel")
+ if not isinstance(wheel, str):
+ pytest.fail("--wheel is required for release wheel checks")
+ inspected: Final = inspect_wheel(Path(wheel))
+ if isinstance(inspected, WheelInspectionError):
+ pytest.fail(inspected.message)
+ return inspected
diff --git a/tests/wheel/test_linux_native_wheel.py b/tests/wheel/test_linux_native_wheel.py
new file mode 100644
index 00000000000..1a70595b717
--- /dev/null
+++ b/tests/wheel/test_linux_native_wheel.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from pytest_ci_report import CIMetric
+from verify_linux_native_wheel import (
+ EXPECTED_ABI_TAG,
+ EXPECTED_PLATFORM_TAG,
+ EXPECTED_PYTHON_TAG,
+ WheelFacts,
+)
+
+
+def test_python_tag_is_cp310(facts: WheelFacts) -> None:
+ assert facts.python_tag == EXPECTED_PYTHON_TAG, f"unexpected Python tag: {facts.python_tag}"
+
+
+def test_abi_tag_is_abi3(facts: WheelFacts) -> None:
+ assert facts.abi_tag == EXPECTED_ABI_TAG, f"unexpected ABI tag: {facts.abi_tag}"
+
+
+def test_platform_tag_is_linux_x86_64(facts: WheelFacts) -> None:
+ assert facts.platform_tag == EXPECTED_PLATFORM_TAG, f"unexpected platform tag: {facts.platform_tag}"
+
+
+def test_dist_info_directory_matches_filename(facts: WheelFacts) -> None:
+ assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,)), (
+ f"unexpected dist-info directories: {facts.dist_info_directories}"
+ )
+
+
+def test_required_dist_info_files_are_present_once(facts: WheelFacts) -> None:
+ assert all(count == 1 for count in facts.dist_info_file_counts.values()), (
+ f"required dist-info file counts are invalid: {facts.dist_info_file_counts}"
+ )
+
+
+def test_metadata_tags_match_filename(facts: WheelFacts) -> None:
+ assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags) and (
+ frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags
+ ), f"WHEEL tags do not match filename: {facts.wheel_metadata_tags}"
+
+
+def test_debug_sections_are_absent(facts: WheelFacts) -> None:
+ assert not facts.debug_sections, f"native extension contains debug sections: {facts.debug_sections}"
+
+
+def test_static_symbol_table_is_absent(facts: WheelFacts) -> None:
+ assert facts.static_symbol_table_absent, "native extension contains a static symbol table"
+
+
+def test_python_extension_entry_point_is_present(facts: WheelFacts) -> None:
+ assert facts.extension_entry_point_present, "native extension does not export PyInit__native"
+
+
+def test_native_module_loads(facts: WheelFacts) -> None:
+ assert facts.native_module_loads, "native extension cannot be loaded"
+
+
+def test_production_module_omits_panic_hook(facts: WheelFacts) -> None:
+ assert facts.panic_test_hook_absent, "production native extension exposes _panic_for_test"
+
+
+def test_native_extension_does_not_exceed_48_mb(facts: WheelFacts) -> None:
+ assert facts.native_size <= 48_000_000, (
+ f"native extension exceeds 48 MB: {facts.native_size / 1_000_000:.2f} MB"
+ )
+
+
+def test_tokenizer_vocabularies_are_not_duplicated(facts: WheelFacts) -> None:
+ assert not facts.duplicated_vocabularies, (
+ f"tokenizer vocabularies are duplicated in the native extension: {facts.duplicated_vocabularies}"
+ )
+
+
+def test_wheel_contents_are_valid(facts: WheelFacts) -> None:
+ assert not facts.unexpected_members, f"wheel contains unexpected build artifacts: {facts.unexpected_members}"
+
+
+def test_build_report(facts: WheelFacts, ci_metric: CIMetric) -> None:
+ assert 0 < facts.native_size <= facts.uncompressed_size
+ ci_metric("Compressed wheel", round(facts.compressed_size / 1_000_000, 2), "MB")
+ ci_metric("Uncompressed wheel", round(facts.uncompressed_size / 1_000_000, 2), "MB")
+ ci_metric("Native extension", round(facts.native_size / 1_000_000, 2), "MB")
+ ci_metric("Native share", round(facts.native_size / facts.uncompressed_size * 100, 2), "%")
+ ci_metric("Commit", facts.commit_sha)
+ ci_metric("Platform", facts.platform_tag)
+ ci_metric("Python ABI", f"{facts.python_tag}-{facts.abi_tag}")
+ ci_metric("Rust compiler", facts.rustc_version)
+ ci_metric("Maturin", facts.maturin_version)
+ ci_metric("Cargo profile", "release")
From 58b9c6bfa8afba6c1620458fc6d1c13a84fc7dc6 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 19 Oct 2026 18:46:46 +0000
Subject: [PATCH 3/5] ci: polish pytest report summaries
---
.github/scripts/render_pytest_report.py | 89 ++++++++++++-------
.../test_pytest_ci_report.py | 53 ++++++++++-
tests/unit/test_unit_shard_missing_paths.py | 5 +-
3 files changed, 112 insertions(+), 35 deletions(-)
diff --git a/.github/scripts/render_pytest_report.py b/.github/scripts/render_pytest_report.py
index 7da11445643..5aa51fb5cdc 100644
--- a/.github/scripts/render_pytest_report.py
+++ b/.github/scripts/render_pytest_report.py
@@ -7,6 +7,7 @@ import sys
from collections import Counter
from dataclasses import dataclass
from itertools import accumulate, groupby
+from os.path import commonpath
from pathlib import Path
from typing import Final, cast
@@ -43,9 +44,9 @@ def _outcome(row: dict[str, object]) -> Outcome:
return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"]))
-def _cell(value: str | float) -> str:
+def _cell(value: str | float, limit: int = 240) -> str:
raw: Final = str(value)
- clipped: Final = raw[:240] + ("..." if len(raw) > 240 else "")
+ clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "")
return (
html.escape(" ".join(clipped.split()), quote=True)
.replace("|", "|")
@@ -75,18 +76,36 @@ def _metric_row(row: dict[str, object]) -> str:
value: Final = row["value"]
if not isinstance(value, (str, int, float)):
raise ValueError("invalid metric value")
- return (
- "| "
- + " | ".join(
- _cell(value)
- for value in (
- _string(row["name"]),
- value,
- _string(row["unit"]),
- _string(row["nodeid"]),
- )
+ unit: Final = _string(row["unit"])
+ measured: Final = _cell(f"{value} {unit}".strip())
+ formatted: Final = f"{measured}" if isinstance(value, (int, float)) else measured
+ return f"| {_cell(_string(row['name']))} | {formatted} |"
+
+
+def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str:
+ return "\n".join(
+ (
+ "| Metric | Value |",
+ "| --- | --- |",
+ _limited(tuple(_metric_row(row) for row in rows)),
+ "",
+ "",
+ "Metric source
",
+ "",
+ f"{_cell(nodeid)}",
+ "",
+ " ",
+ "",
)
- + " |"
+ )
+
+
+def _expected_detail(outcome: Outcome) -> str:
+ path, _, test = outcome.nodeid.partition("::")
+ return (
+ f"- **{outcome.category}**: {_cell(test or path)}\n\n"
+ f" {_cell(outcome.reason, 1000)}\n\n"
+ f" {_cell(path)}\n"
)
@@ -104,24 +123,36 @@ def render(data: dict[str, object], title: str) -> str:
or "No test outcomes recorded"
)
files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed")
+ prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "."
grouped: Final = groupby(
sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0]
)
- file_rows: Final = tuple(_file_row(path, tuple(group)) for path, group in grouped if path in files)
- table: Final = "\n".join(
- (
- "| File | passed | failed | error | skipped | xfailed | xpassed |",
- "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
- _limited(file_rows),
+ file_rows: Final = tuple(
+ _file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files
+ )
+ table: Final = (
+ "\n".join(
+ (
+ "### Files with non-pass outcomes",
+ "",
+ f"Paths relative to {_cell(prefix)}/\n" if prefix != "." else "",
+ "| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+ _limited(file_rows),
+ )
)
+ if file_rows
+ else ""
+ )
+ metric_groups: Final = groupby(
+ sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"])
)
metric_table: Final = (
"\n".join(
(
+ "### Metrics",
"",
- "| Metric | Value | Unit | Test |",
- "| --- | --- | --- | --- |",
- _limited(tuple(_metric_row(row) for row in metrics)),
+ _limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)),
)
)
if metrics
@@ -138,14 +169,9 @@ def render(data: dict[str, object], title: str) -> str:
(
"",
"",
- "Xfail and xpass reasons
",
+ f"Xfail and xpass reasons ({len(expected)})
",
"",
- _limited(
- tuple(
- f"- {outcome.category}: {_cell(outcome.nodeid)}: {_cell(outcome.reason)}"
- for outcome in expected
- )
- ),
+ _limited(tuple(_expected_detail(outcome) for outcome in expected)),
"",
" ",
)
@@ -153,7 +179,10 @@ def render(data: dict[str, object], title: str) -> str:
if expected
else ""
)
- return "\n".join((f"## {_cell(title)}", "", totals, "", table, metric_table, details, ""))
+ return (
+ "\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part)
+ + "\n"
+ )
def main() -> None:
diff --git a/tests/code_coverage_tests/test_pytest_ci_report.py b/tests/code_coverage_tests/test_pytest_ci_report.py
index fdd72478998..ec442b9a711 100644
--- a/tests/code_coverage_tests/test_pytest_ci_report.py
+++ b/tests/code_coverage_tests/test_pytest_ci_report.py
@@ -149,6 +149,7 @@ def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, worke
)
assert ", ".join(terminal) in rendered.stdout
assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout)
+ assert "Paths relative to" not in html.unescape(rendered.stdout)
assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug")
@@ -173,9 +174,7 @@ def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, worke
terminal_summary: Final = next(
line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line)
)
- terminal_counts: Final = re.findall(
- r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary
- )
+ terminal_counts: Final = re.findall(r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary)
assert terminal_counts == ["1 error"]
rendered: Final = render(report)
assert rendered.returncode == 0, rendered.stderr
@@ -264,13 +263,59 @@ def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str)
)
result: Final = render(report)
assert result.returncode == 0
- visible: Final = result.stdout.count("- xfailed:")
+ visible: Final = result.stdout.count("- **xfailed**:")
assert visible <= 500
assert f"and {600 - visible} more" in result.stdout
assert len(result.stdout.encode()) < 1_000_000
assert "" not in result.stdout
+def test_passing_metrics_are_compact_and_preserve_sources(tmp_path: Path) -> None:
+ report: Final = tmp_path / "report.json"
+ _ = report.write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "outcomes": [{"nodeid": "tests/wheel/test_build.py::test_sizes", "category": "passed", "reason": ""}],
+ "metrics": [
+ {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Wheel", "value": 20.5, "unit": "MB"},
+ {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Native", "value": 30, "unit": "MB"},
+ ],
+ }
+ )
+ )
+ rendered: Final = render(report)
+ assert rendered.returncode == 0, rendered.stderr
+ readable: Final = html.unescape(rendered.stdout)
+ assert "**1 passed**" in readable
+ assert "| Wheel | 20.5 MB |" in readable
+ assert "| Native | 30 MB |" in readable
+ assert readable.count("tests/wheel/test_build.py::test_sizes") == 1
+ assert "Files with non-pass outcomes" not in readable
+
+
+def test_file_rows_shorten_shared_directory_without_losing_paths(tmp_path: Path) -> None:
+ report: Final = tmp_path / "report.json"
+ _ = report.write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "metrics": [],
+ "outcomes": [
+ {"nodeid": "tests/unit/proxy/test_first.py::test_one", "category": "skipped", "reason": ""},
+ {"nodeid": "tests/unit/proxy/test_second.py::test_two", "category": "failed", "reason": ""},
+ ],
+ }
+ )
+ )
+ rendered: Final = render(report)
+ readable: Final = html.unescape(rendered.stdout)
+ assert rendered.returncode == 0, rendered.stderr
+ assert "Paths relative to tests/unit/proxy/" in readable
+ assert "| test_first.py | 0 | 0 | 0 | 1 | 0 | 0 |" in readable
+ assert "| test_second.py | 0 | 1 | 0 | 0 | 0 | 0 |" in readable
+
+
def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None:
result: Final = run_suite(
tmp_path,
diff --git a/tests/unit/test_unit_shard_missing_paths.py b/tests/unit/test_unit_shard_missing_paths.py
index 725551e19d3..60cc1390833 100644
--- a/tests/unit/test_unit_shard_missing_paths.py
+++ b/tests/unit/test_unit_shard_missing_paths.py
@@ -1,3 +1,4 @@
+import json
import os
import subprocess
import sys
@@ -28,7 +29,9 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl
shim_dir.mkdir()
(shim_dir / "uv").write_text(_UV_SHIM)
(shim_dir / "uv").chmod(0o755)
- (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n")
+ (tmp_path / "pyproject.toml").write_text(
+ f"[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\npythonpath = [{json.dumps(str(_REPO_ROOT / 'scripts'))}]\n"
+ )
return subprocess.run(
("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()),
cwd=tmp_path,
From d10691751fea0015159d99176d8f5cc6fa361497 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 19 Oct 2026 18:48:17 +0000
Subject: [PATCH 4/5] test: demonstrate failing wheel checks still report
outcomes
---
tests/wheel/test_linux_native_wheel.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/tests/wheel/test_linux_native_wheel.py b/tests/wheel/test_linux_native_wheel.py
index 1a70595b717..fa724dd7568 100644
--- a/tests/wheel/test_linux_native_wheel.py
+++ b/tests/wheel/test_linux_native_wheel.py
@@ -60,9 +60,7 @@ def test_production_module_omits_panic_hook(facts: WheelFacts) -> None:
def test_native_extension_does_not_exceed_48_mb(facts: WheelFacts) -> None:
- assert facts.native_size <= 48_000_000, (
- f"native extension exceeds 48 MB: {facts.native_size / 1_000_000:.2f} MB"
- )
+ assert facts.native_size <= 1, "Deliberate CI proof: native extension limit set to one byte"
def test_tokenizer_vocabularies_are_not_duplicated(facts: WheelFacts) -> None:
From 2880824203d706b4a1ce535278d84438a3334da4 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 19 Oct 2026 18:58:03 +0000
Subject: [PATCH 5/5] Revert "test: demonstrate failing wheel checks still
report outcomes"
This reverts commit d10691751fea0015159d99176d8f5cc6fa361497.
---
tests/wheel/test_linux_native_wheel.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tests/wheel/test_linux_native_wheel.py b/tests/wheel/test_linux_native_wheel.py
index fa724dd7568..1a70595b717 100644
--- a/tests/wheel/test_linux_native_wheel.py
+++ b/tests/wheel/test_linux_native_wheel.py
@@ -60,7 +60,9 @@ def test_production_module_omits_panic_hook(facts: WheelFacts) -> None:
def test_native_extension_does_not_exceed_48_mb(facts: WheelFacts) -> None:
- assert facts.native_size <= 1, "Deliberate CI proof: native extension limit set to one byte"
+ assert facts.native_size <= 48_000_000, (
+ f"native extension exceeds 48 MB: {facts.native_size / 1_000_000:.2f} MB"
+ )
def test_tokenizer_vocabularies_are_not_duplicated(facts: WheelFacts) -> None: