diff --git a/.github/actions/pytest-report/action.yml b/.github/actions/pytest-report/action.yml new file mode 100644 index 00000000000..4c5c342cec3 --- /dev/null +++ b/.github/actions/pytest-report/action.yml @@ -0,0 +1,38 @@ +name: "Report pytest outcomes" +description: >- + Append non-gating pytest outcomes and metrics to the job summary, and optionally + retain the raw report for later comparison. + +inputs: + report: + description: "Path to the pytest report JSON" + required: true + title: + description: "Heading shown in the job summary" + required: true + artifact-name: + description: "Optional artifact name for retaining the raw report" + required: false + default: "" + +runs: + using: composite + steps: + - name: Render pytest report + if: always() + continue-on-error: true + shell: bash + env: + REPORT: ${{ inputs.report }} + TITLE: ${{ inputs.title }} + run: python3 "$GITHUB_ACTION_PATH/../../scripts/render_pytest_report.py" --title "$TITLE" "$REPORT" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload pytest report + if: ${{ always() && inputs.artifact-name != '' }} + continue-on-error: true + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.report }} + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/scripts/render_pytest_report.py b/.github/scripts/render_pytest_report.py new file mode 100644 index 00000000000..5aa51fb5cdc --- /dev/null +++ b/.github/scripts/render_pytest_report.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import argparse +import html +import json +import sys +from collections import Counter +from dataclasses import dataclass +from itertools import accumulate, groupby +from os.path import commonpath +from pathlib import Path +from typing import Final, cast + +TOTAL_ORDER: Final = ("failed", "passed", "skipped", "xfailed", "xpassed", "error") +COLUMNS: Final = ("passed", "failed", "error", "skipped", "xfailed", "xpassed") + + +@dataclass(frozen=True, slots=True) +class Outcome: + nodeid: str + category: str + reason: str + + +def _object(value: object) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError("expected an object") + return cast(dict[str, object], value) + + +def _array(value: object) -> tuple[dict[str, object], ...]: + if not isinstance(value, list): + raise ValueError("expected an array") + return tuple(_object(row) for row in cast(list[object], value)) + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise ValueError("expected a string") + return value + + +def _outcome(row: dict[str, object]) -> Outcome: + return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"])) + + +def _cell(value: str | float, limit: int = 240) -> str: + raw: Final = str(value) + clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "") + return ( + html.escape(" ".join(clipped.split()), quote=True) + .replace("|", "|") + .replace("`", "`") + .replace("[", "[") + .replace("]", "]") + .replace("*", "*") + .replace("_", "_") + .replace("\\", "\") + ) + + +def _limited(rows: tuple[str, ...]) -> str: + candidates: Final = rows[:500] + sizes: Final = accumulate(len(row.encode("utf-8")) + 1 for row in candidates) + visible: Final = tuple(row for row, size in zip(candidates, sizes) if size <= 200_000) + remainder: Final = len(rows) - len(visible) + return "\n".join(visible + ((f"\nand {remainder} more",) if remainder else ())) + + +def _file_row(path: str, outcomes: tuple[Outcome, ...]) -> str: + counts: Final = Counter(outcome.category for outcome in outcomes) + return "| " + _cell(path) + " | " + " | ".join(str(counts[column]) for column in COLUMNS) + " |" + + +def _metric_row(row: dict[str, object]) -> str: + value: Final = row["value"] + if not isinstance(value, (str, int, float)): + raise ValueError("invalid metric value") + unit: Final = _string(row["unit"]) + measured: Final = _cell(f"{value} {unit}".strip()) + formatted: Final = f"{measured}" if isinstance(value, (int, float)) else measured + return f"| {_cell(_string(row['name']))} | {formatted} |" + + +def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str: + return "\n".join( + ( + "| Metric | Value |", + "| --- | --- |", + _limited(tuple(_metric_row(row) for row in rows)), + "", + "
", + "Metric source", + "", + f"{_cell(nodeid)}", + "", + "
", + "", + ) + ) + + +def _expected_detail(outcome: Outcome) -> str: + path, _, test = outcome.nodeid.partition("::") + return ( + f"- **{outcome.category}**: {_cell(test or path)}\n\n" + f" {_cell(outcome.reason, 1000)}\n\n" + f" {_cell(path)}\n" + ) + + +def render(data: dict[str, object], title: str) -> str: + outcomes: Final = tuple(_outcome(row) for row in _array(data["outcomes"])) + metrics: Final = _array(data["metrics"]) + counts: Final = Counter(outcome.category for outcome in outcomes) + extras: Final = tuple(sorted(set(counts) - set(TOTAL_ORDER))) + totals: Final = ( + ", ".join( + f"{counts[category]} {'errors' if category == 'error' and counts[category] != 1 else category}" + for category in TOTAL_ORDER + extras + if counts[category] + ) + or "No test outcomes recorded" + ) + files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed") + prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "." + grouped: Final = groupby( + sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0] + ) + file_rows: Final = tuple( + _file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files + ) + table: Final = ( + "\n".join( + ( + "### Files with non-pass outcomes", + "", + f"Paths relative to {_cell(prefix)}/\n" if prefix != "." else "", + "| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + _limited(file_rows), + ) + ) + if file_rows + else "" + ) + metric_groups: Final = groupby( + sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"]) + ) + metric_table: Final = ( + "\n".join( + ( + "### Metrics", + "", + _limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)), + ) + ) + if metrics + else "" + ) + expected: Final = tuple( + sorted( + (outcome for outcome in outcomes if outcome.category in ("xfailed", "xpassed")), + key=lambda outcome: (outcome.category != "xpassed", outcome.nodeid), + ) + ) + details: Final = ( + "\n".join( + ( + "", + "
", + f"Xfail and xpass reasons ({len(expected)})", + "", + _limited(tuple(_expected_detail(outcome) for outcome in expected)), + "", + "
", + ) + ) + if expected + else "" + ) + return ( + "\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part) + + "\n" + ) + + +def main() -> None: + parser: Final = argparse.ArgumentParser() + _ = parser.add_argument("--title", required=True) + _ = parser.add_argument("report", type=Path) + arguments: Final = parser.parse_args() + path: Final = cast(Path, arguments.report) + title: Final = cast(str, arguments.title) + try: + data: Final = _object(cast(object, json.loads(path.read_text(encoding="utf-8")))) + if data.get("version") != 1: + sys.stdout.write("Unsupported report version; no report was produced\n") + return + sys.stdout.write(render(data, title)) + except (OSError, ValueError, KeyError, TypeError, RecursionError): + sys.stdout.write("No report was produced (missing or unreadable report)\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 4c4cb399dce..99f282e7bc3 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -6,7 +6,8 @@ import re import subprocess import sys import zipfile -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping +from dataclasses import dataclass from itertools import product from pathlib import Path, PurePosixPath from types import MappingProxyType, ModuleType @@ -17,6 +18,36 @@ EXPECTED_ABI_TAG: Final = "abi3" EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" +@dataclass(frozen=True, slots=True) +class WheelInspectionError: + message: str + + +@dataclass(frozen=True, slots=True) +class WheelFacts: + python_tag: str + abi_tag: str + platform_tag: str + expected_dist_info_directory: str + dist_info_directories: frozenset[str] + dist_info_file_counts: Mapping[str, int] + wheel_metadata_tags: tuple[str, ...] + expanded_filename_tags: frozenset[str] + compressed_size: int + uncompressed_size: int + native_size: int + debug_sections: tuple[str, ...] + static_symbol_table_absent: bool + extension_entry_point_present: bool + native_module_loads: bool + panic_test_hook_absent: bool + duplicated_vocabularies: tuple[str, ...] + unexpected_members: tuple[str, ...] + commit_sha: str + rustc_version: str + maturin_version: str + + class CommandRunner(Protocol): def __call__( self, @@ -65,31 +96,22 @@ def _load_native_module(native_path: Path) -> ModuleType | None: return native_module -def main( - argv: Sequence[str] | None = None, +def inspect_wheel( + wheel: Path, environment: Mapping[str, str] | None = None, load_native_module: Callable[[Path], ModuleType | None] = _load_native_module, run_command: CommandRunner = _run_command, -) -> int: - arguments: Final = tuple(sys.argv if argv is None else argv) +) -> WheelFacts | WheelInspectionError: resolved_environment: Final = os.environ if environment is None else environment - if len(arguments) != 2: - sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") - return 2 - - wheel: Final = Path(arguments[1]) wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) if len(wheel_tags) != 4: - sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") - return 1 + return WheelInspectionError(f"cannot parse wheel tags from {wheel.name}") wheel_identity: Final = wheel_tags[0].split("-") if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: - sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") - return 1 + return WheelInspectionError(f"unexpected wheel identity: {wheel_tags[0]}") expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info" - expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,)) python_tag: Final = wheel_tags[1] abi_tag: Final = wheel_tags[2] platform_tag: Final = wheel_tags[3] @@ -121,8 +143,7 @@ def main( if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") ) if len(native_members) != 1: - sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") - return 1 + return WheelInspectionError(f"expected one native extension, found {len(native_members)}") unexpected_members: Final = tuple( member.filename @@ -145,10 +166,6 @@ def main( and archive.read(member) in native_bytes ) - wheel_metadata_tags_match: Final = ( - len(wheel_metadata_tags) == len(expanded_filename_tags) - and frozenset(wheel_metadata_tags) == expanded_filename_tags - ) commit_sha: Final = resolved_environment.get( "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown") ) @@ -161,39 +178,9 @@ def main( pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) if maturin_match is None: - sys.stderr.write("build-system does not pin an exact Maturin version\n") - return 1 + return WheelInspectionError("build-system does not pin an exact Maturin version") maturin_version: Final = maturin_match.group(1) - native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 - size_report: Final = "\n".join( - ( - "## Native wheel build report", - "", - "| Build | Value |", - "| --- | --- |", - f"| Commit | `{commit_sha}` |", - f"| Platform | `{platform_tag}` |", - f"| Python ABI | `{python_tag}-{abi_tag}` |", - f"| Rust compiler | `{rustc_version}` |", - f"| Maturin | `{maturin_version}` |", - "| Cargo profile | `release` |", - "", - "| Artifact | Size |", - "| --- | ---: |", - f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", - f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", - f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", - f"| Native share | {native_percentage:.2f}% |", - "", - ) - ) - summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") - if summary_path is None: - sys.stdout.write(size_report) - else: - Path(summary_path).write_text(size_report) - sections: Final = run_command( ("readelf", "--sections", "--wide", str(native_path)), check=True, @@ -201,7 +188,6 @@ def main( text=True, ).stdout debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) - debug_sections_absent: Final = not debug_sections static_symbol_table_absent: Final = ".symtab" not in sections dynamic_symbols: Final = run_command( @@ -214,80 +200,26 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 48_000_000 - native_size_within_limit: Final = native_member.file_size <= native_size_limit - validations: Final = ( - (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), - (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), - (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), - ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), - ( - "Required dist-info files are present exactly once", - all(count == 1 for count in dist_info_file_counts.values()), - ), - ("Wheel metadata tags match the filename", wheel_metadata_tags_match), - ("Debug sections are absent", debug_sections_absent), - ("Static symbol table is absent", static_symbol_table_absent), - ("Python extension entry point is present", extension_entry_point_present), - ("Native module loads", native_module_loads), - ("Production module omits the panic test hook", panic_test_hook_absent), - (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), - ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), - ("Wheel contents are valid", not unexpected_members), + return WheelFacts( + python_tag=python_tag, + abi_tag=abi_tag, + platform_tag=platform_tag, + expected_dist_info_directory=expected_dist_info_directory, + dist_info_directories=dist_info_directories, + dist_info_file_counts=dist_info_file_counts, + wheel_metadata_tags=wheel_metadata_tags, + expanded_filename_tags=expanded_filename_tags, + compressed_size=wheel.stat().st_size, + uncompressed_size=uncompressed_wheel_size, + native_size=native_member.file_size, + debug_sections=debug_sections, + static_symbol_table_absent=static_symbol_table_absent, + extension_entry_point_present=extension_entry_point_present, + native_module_loads=native_module_loads, + panic_test_hook_absent=panic_test_hook_absent, + duplicated_vocabularies=duplicated_vocabularies, + unexpected_members=unexpected_members, + commit_sha=commit_sha, + rustc_version=rustc_version, + maturin_version=maturin_version, ) - - verified_report: Final = size_report + "\n".join( - ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") - + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) - + ("",) - ) - if summary_path is not None: - Path(summary_path).write_text(verified_report) - - invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) - validation_errors: Final = tuple( - message - for failed, message in ( - (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), - (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), - (not extension_entry_point_present, "native extension does not export PyInit__native"), - ( - python_tag != EXPECTED_PYTHON_TAG, - f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", - ), - (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), - ( - platform_tag != EXPECTED_PLATFORM_TAG, - f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", - ), - ( - dist_info_directories != expected_dist_info_directories, - f"unexpected dist-info directories: expected {expected_dist_info_directory}, " - f"found {', '.join(sorted(dist_info_directories))}", - ), - (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), - ( - not invalid_dist_info_files and not wheel_metadata_tags_match, - f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " - f"found {', '.join(sorted(wheel_metadata_tags))}", - ), - ( - native_module is not None and not panic_test_hook_absent, - "production native module exposes _panic_for_test", - ), - ( - not native_size_within_limit, - f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: " - f"{native_member.file_size / 1_000_000:.2f} MB", - ), - (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), - ) - if failed - ) - sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) - - return 0 if all(passed for _, passed in validations) else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index cbeea71ad0f..3890658fe08 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -554,6 +554,9 @@ jobs: export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)" uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py + - name: test_pytest_ci_report + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_pytest_ci_report.py + - name: Check merge smoke harness run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index cf9fed192f6..3942ae66615 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -31,6 +31,10 @@ on: - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" + - ".github/actions/pytest-report/**" + - ".github/scripts/render_pytest_report.py" + - "scripts/pytest_ci_report.py" + - "tests/wheel/**" - "tests/unit/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" pull_request: @@ -66,6 +70,10 @@ on: - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" + - ".github/actions/pytest-report/**" + - ".github/scripts/render_pytest_report.py" + - "scripts/pytest_ci_report.py" + - "tests/wheel/**" - "tests/unit/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" @@ -211,14 +219,34 @@ jobs: - run: uv build --wheel --out-dir dist - - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + - name: Check native wheel + run: | + set -- dist/*.whl + [ "$#" -eq 1 ] + uv run --no-project --with pytest==9.0.3 python -m pytest tests/wheel --wheel "$1" -p pytest_ci_report --ci-report=wheel-report.json env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Report native wheel outcomes + if: always() + uses: ./.github/actions/pytest-report + with: + report: wheel-report.json + title: Native wheel + artifact-name: wheel-report-${{ github.run_id }}-${{ github.run_attempt }} + - run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl - name: Run pytest tests/test_litellm_rust with the compiled extension - run: make test-rust-extension + run: make test-rust-extension PYTEST_REPORT_ARGS="-p pytest_ci_report --ci-report=rust-extension-report.json" + + - name: Report Rust extension outcomes + if: always() + uses: ./.github/actions/pytest-report + with: + report: rust-extension-report.json + title: Rust extension + artifact-name: rust-extension-report-${{ github.run_id }}-${{ github.run_attempt }} - run: >- uv build --wheel --out-dir panic-dist diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 0a490bb8f45..fa54ac09a93 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1085,7 +1085,7 @@ jobs: xdist_args=(-n "${WORKERS}" --dist="${DIST}") fi set +e - uv run --no-sync pytest "${pytest_args[@]}" \ + uv run --no-sync pytest -p pytest_ci_report --ci-report=pytest-report.json "${pytest_args[@]}" \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ "${xdist_args[@]}" \ @@ -1115,6 +1115,14 @@ jobs: name: coverage-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} path: coverage.xml retention-days: 1 + + - name: Report test outcomes + if: always() && steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/pytest-report + with: + report: pytest-report.json + title: ${{ matrix.shard }} + artifact-name: pytest-report-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} ui-unit: name: ui-unit permissions: diff --git a/Makefile b/Makefile index 48df3c8e215..cf86f5b1f78 100644 --- a/Makefile +++ b/Makefile @@ -279,6 +279,8 @@ pre-commit: @$(MAKE) check # Testing targets +PYTEST_REPORT_ARGS ?= + test-rust-extension: @temporary=$$(mktemp -d) && \ trap 'rm -rf "$$temporary"' EXIT HUP INT TERM && \ @@ -291,7 +293,7 @@ test-rust-extension: --mypy-config-file tests/unit/rust_bridge/stubtest.ini \ litellm.rust_bridge._native && \ LITELLM_LOCAL_MODEL_COST_MAP=True \ - "$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust + "$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension $(PYTEST_REPORT_ARGS) tests/test_litellm_rust rust-sqlx-prepare: cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare diff --git a/scripts/pytest_ci_report.py b/scripts/pytest_ci_report.py new file mode 100644 index 00000000000..f63de455f8f --- /dev/null +++ b/scripts/pytest_ci_report.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Final, Protocol, cast + +import pytest + + +class CIMetric(Protocol): + def __call__(self, name: str, value: str | float, unit: str = "") -> None: ... + + +@dataclass(frozen=True, slots=True) +class Outcome: + nodeid: str + when: str + category: str + reason: str + + +@dataclass(frozen=True, slots=True) +class Metric: + nodeid: str + name: str + value: str | float + unit: str + + +@dataclass(frozen=True, slots=True) +class Report: + outcomes: tuple[Outcome, ...] = () + metrics: tuple[Metric, ...] = () + version: int = 1 + + +REPORT: Final = pytest.StashKey[Report]() + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption("--ci-report", default=None, help="Write non-gating CI outcomes and metrics as JSON") + + +def pytest_configure(config: pytest.Config) -> None: + path: Final[object] = config.getoption("ci_report") + if not isinstance(path, str) or hasattr(config, "workerinput"): + return + config.stash[REPORT] = Report() + config.pluginmanager.register(CIReporter(config, Path(path)), "ci-report-writer") + + +def _reason(report: pytest.TestReport | pytest.CollectReport) -> str: + xfail: Final[object] = getattr(report, "wasxfail", None) + if isinstance(xfail, str): + return xfail + if report.skipped and isinstance(report.longrepr, tuple): + return str(report.longrepr[2]) + if isinstance(report, pytest.CollectReport) and report.failed: + return str(report.longrepr) + return "" + + +def _metric(nodeid: str, prop: object) -> Metric | None: + if not isinstance(prop, (tuple, list)) or len(prop) != 2 or prop[0] != "ci_metric": + return None + value: Final[object] = prop[1] + if not isinstance(value, (tuple, list)) or len(value) != 3: + return None + name, amount, unit = cast(tuple[object, object, object], value) + if isinstance(name, str) and isinstance(amount, (str, int, float)) and isinstance(unit, str): + return Metric(nodeid, name, amount, unit) + return None + + +@dataclass(frozen=True, slots=True) +class CIReporter: + config: pytest.Config + path: Path + + def pytest_collectreport(self, report: pytest.CollectReport) -> None: + status: Final = cast( + tuple[str, str, object], + self.config.hook.pytest_report_teststatus( + report=report, + config=self.config, + ), + ) + category: Final = status[0] + if category not in ("error", "skipped"): + return + previous: Final = self.config.stash[REPORT] + self.config.stash[REPORT] = Report( + previous.outcomes + (Outcome(report.nodeid, "collect", category, _reason(report)),), + previous.metrics, + ) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + status: Final = cast( + tuple[str, str, object], + self.config.hook.pytest_report_teststatus( + report=report, + config=self.config, + ), + ) + category: Final = status[0] + previous: Final = self.config.stash[REPORT] + outcomes: Final = previous.outcomes + ( + (Outcome(report.nodeid, report.when, category, _reason(report)),) if category not in ("", "rerun") else () + ) + metrics: Final = ( + tuple(metric for metric in previous.metrics if metric.nodeid != report.nodeid) + + tuple(metric for prop in report.user_properties if (metric := _metric(report.nodeid, prop)) is not None) + if report.when == "teardown" + else previous.metrics + ) + self.config.stash[REPORT] = Report(outcomes, metrics) + + def pytest_sessionfinish(self) -> None: + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + _ = self.path.write_text(json.dumps(asdict(self.config.stash[REPORT])), encoding="utf-8") + except (OSError, ValueError) as error: + sys.stderr.write(f"Could not write CI report: {error}\n") + + +@pytest.fixture +def ci_metric(request: pytest.FixtureRequest) -> CIMetric: + def record(name: str, value: str | float, unit: str = "") -> None: + if request.config.getoption("ci_report") is not None: + request.node.user_properties.append(("ci_metric", (name, value, unit))) + + return record diff --git a/tests/code_coverage_tests/test_pytest_ci_report.py b/tests/code_coverage_tests/test_pytest_ci_report.py new file mode 100644 index 00000000000..ec442b9a711 --- /dev/null +++ b/tests/code_coverage_tests/test_pytest_ci_report.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import html +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest +from pydantic import BaseModel, ConfigDict, TypeAdapter + +ROOT: Final = Path(__file__).resolve().parents[2] +RENDERER: Final = ROOT / ".github/scripts/render_pytest_report.py" +SUITE: Final = """ +from pathlib import Path +import pytest + +def test_pass(ci_metric): + ci_metric("Size", 41.2, "MB") + +def test_fail(): + assert False, "permanent failure" + +def test_skip(): + pytest.skip("not available") + +@pytest.mark.xfail(reason="known bug") +def test_xfail(): + assert False + +@pytest.mark.xfail(reason="fixed bug") +def test_xpass(): + pass + +def test_retry(): + marker = Path(__file__).with_name("attempted") + first = not marker.exists() + marker.touch() + assert not first, "retry me" + +@pytest.fixture +def teardown_error(): + yield + raise RuntimeError("cleanup failed") + +def test_teardown(teardown_error): + pass +""" + + +class ReportOutcome(BaseModel): + model_config = ConfigDict(frozen=True) + + nodeid: str + when: str + category: str + reason: str + + +class ReportMetric(BaseModel): + model_config = ConfigDict(frozen=True) + + nodeid: str + name: str + value: str | float + unit: str + + +class ReportData(BaseModel): + model_config = ConfigDict(frozen=True) + + version: int + outcomes: tuple[ReportOutcome, ...] + metrics: tuple[ReportMetric, ...] + + +REPORT_DATA: Final = TypeAdapter(ReportData) + + +def run_suite(directory: Path, source: str = SUITE, *args: str) -> subprocess.CompletedProcess[str]: + directory.mkdir(exist_ok=True) + _ = (directory / "test_sample.py").write_text(source) + return subprocess.run( + ( + sys.executable, + "-P", + "-m", + "pytest", + "-c", + "/dev/null", + f"--rootdir={directory}", + "--noconftest", + "-p", + "pytest_ci_report", + "-p", + "no:cacheprovider", + "--color=no", + "-rA", + *args, + "test_sample.py", + ), + cwd=directory, + env={**os.environ, "PYTHONPATH": str(ROOT / "scripts"), "PYTEST_ADDOPTS": ""}, + capture_output=True, + text=True, + check=False, + timeout=90, + ) + + +def render(path: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + (sys.executable, "-P", str(RENDERER), "--title", "CI outcomes", str(path)), + capture_output=True, + text=True, + check=False, + timeout=10, + ) + + +@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist")) +def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, workers: tuple[str, ...]) -> None: + report: Final = tmp_path / "report.json" + result: Final = run_suite( + tmp_path, SUITE, "--ci-report", str(report), "--reruns", "1", "--only-rerun", "retry me", *workers + ) + assert result.returncode == 1, result.stdout + result.stderr + data: Final = REPORT_DATA.validate_json(report.read_text()) + assert data.version == 1 + assert sorted((row.nodeid.split("::")[-1], row.when, row.category, row.reason) for row in data.outcomes) == [ + ("test_fail", "call", "failed", ""), + ("test_pass", "call", "passed", ""), + ("test_retry", "call", "passed", ""), + ("test_skip", "call", "skipped", "Skipped: not available"), + ("test_teardown", "call", "passed", ""), + ("test_teardown", "teardown", "error", ""), + ("test_xfail", "call", "xfailed", "known bug"), + ("test_xpass", "call", "xpassed", "fixed bug"), + ] + assert data.metrics == (ReportMetric(nodeid="test_sample.py::test_pass", name="Size", value=41.2, unit="MB"),) + assert tuple(tmp_path.glob("*.json")) == (report,) + rendered: Final = render(report) + assert rendered.returncode == 0, rendered.stderr + terminal: Final = re.findall( + r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", result.stdout.splitlines()[-1] + ) + assert ", ".join(terminal) in rendered.stdout + assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout) + assert "Paths relative to" not in html.unescape(rendered.stdout) + assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug") + + +@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist")) +def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, workers: tuple[str, ...]) -> None: + report: Final = tmp_path / "report.json" + result: Final = run_suite( + tmp_path, + "import missing_ci_test_module\n", + "--ci-report", + str(report), + *workers, + ) + assert result.returncode != 0, result.stdout + result.stderr + data: Final = REPORT_DATA.validate_json(report.read_text()) + assert len(data.outcomes) == 1 + outcome: Final = data.outcomes[0] + assert outcome.nodeid == "test_sample.py" + assert outcome.when == "collect" + assert outcome.category == "error" + assert "missing_ci_test_module" in outcome.reason + terminal_summary: Final = next( + line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line) + ) + terminal_counts: Final = re.findall(r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary) + assert terminal_counts == ["1 error"] + rendered: Final = render(report) + assert rendered.returncode == 0, rendered.stderr + assert ", ".join(terminal_counts) in rendered.stdout + + +def test_collection_skips_are_reported(tmp_path: Path) -> None: + report: Final = tmp_path / "report.json" + result: Final = run_suite( + tmp_path, + "import pytest\npytest.skip('not available', allow_module_level=True)\n", + "--ci-report", + str(report), + ) + data: Final = REPORT_DATA.validate_json(report.read_text()) + assert data.outcomes == ( + ReportOutcome(nodeid="test_sample.py", when="collect", category="skipped", reason="Skipped: not available"), + ) + assert "1 skipped" in result.stdout + rendered: Final = render(report) + assert rendered.returncode == 0, rendered.stderr + assert "1 skipped" in rendered.stdout + + +def test_plugin_loads_with_repository_config_without_pythonpath() -> None: + environment: Final = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + result: Final = subprocess.run( + (sys.executable, "-P", "-m", "pytest", "-p", "pytest_ci_report", "--help"), + cwd=ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + timeout=90, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "--ci-report" in result.stdout + + +def test_maxfail_still_writes_report(tmp_path: Path) -> None: + result: Final = run_suite( + tmp_path, "def test_failure():\n assert False\n", "--ci-report=report.json", "--maxfail=1" + ) + assert result.returncode == 1 + data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text()) + assert data.outcomes == ( + ReportOutcome(nodeid="test_sample.py::test_failure", when="call", category="failed", reason=""), + ) + + +def test_report_is_opt_in(tmp_path: Path) -> None: + result: Final = run_suite(tmp_path, "def test_pass(ci_metric):\n ci_metric('Size', 1)\n") + assert result.returncode == 0, result.stdout + result.stderr + assert not tuple(tmp_path.glob("*.json")) + + +def test_unwritable_report_does_not_fail_tests(tmp_path: Path) -> None: + result: Final = run_suite(tmp_path, "def test_pass():\n pass\n", f"--ci-report={tmp_path}") + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("content", (None, "{", '{"version": 2}', '{"version": 1, "outcomes": null}')) +def test_unavailable_or_unknown_reports_are_non_gating(tmp_path: Path, content: str | None) -> None: + report: Final = tmp_path / "report.json" + if content is not None: + _ = report.write_text(content) + result: Final = render(report) + assert result.returncode == 0, result.stderr + assert "No report was produced" in result.stdout or "Unsupported report version" in result.stdout + + +@pytest.mark.parametrize("reason", ("known bug", "|\n" * 10000), ids=("entry-limit", "byte-limit")) +def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str) -> None: + report: Final = tmp_path / "report.json" + _ = report.write_text( + json.dumps( + { + "version": 1, + "metrics": [], + "outcomes": [ + {"nodeid": f"test_sample.py::test_{index}", "when": "call", "category": "xfailed", "reason": reason} + for index in range(600) + ], + } + ) + ) + result: Final = render(report) + assert result.returncode == 0 + visible: Final = result.stdout.count("- **xfailed**:") + assert visible <= 500 + assert f"and {600 - visible} more" in result.stdout + assert len(result.stdout.encode()) < 1_000_000 + assert "" not in result.stdout + + +def test_passing_metrics_are_compact_and_preserve_sources(tmp_path: Path) -> None: + report: Final = tmp_path / "report.json" + _ = report.write_text( + json.dumps( + { + "version": 1, + "outcomes": [{"nodeid": "tests/wheel/test_build.py::test_sizes", "category": "passed", "reason": ""}], + "metrics": [ + {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Wheel", "value": 20.5, "unit": "MB"}, + {"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Native", "value": 30, "unit": "MB"}, + ], + } + ) + ) + rendered: Final = render(report) + assert rendered.returncode == 0, rendered.stderr + readable: Final = html.unescape(rendered.stdout) + assert "**1 passed**" in readable + assert "| Wheel | 20.5 MB |" in readable + assert "| Native | 30 MB |" in readable + assert readable.count("tests/wheel/test_build.py::test_sizes") == 1 + assert "Files with non-pass outcomes" not in readable + + +def test_file_rows_shorten_shared_directory_without_losing_paths(tmp_path: Path) -> None: + report: Final = tmp_path / "report.json" + _ = report.write_text( + json.dumps( + { + "version": 1, + "metrics": [], + "outcomes": [ + {"nodeid": "tests/unit/proxy/test_first.py::test_one", "category": "skipped", "reason": ""}, + {"nodeid": "tests/unit/proxy/test_second.py::test_two", "category": "failed", "reason": ""}, + ], + } + ) + ) + rendered: Final = render(report) + readable: Final = html.unescape(rendered.stdout) + assert rendered.returncode == 0, rendered.stderr + assert "Paths relative to tests/unit/proxy/" in readable + assert "| test_first.py | 0 | 0 | 0 | 1 | 0 | 0 |" in readable + assert "| test_second.py | 0 | 1 | 0 | 0 | 0 | 0 |" in readable + + +def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None: + result: Final = run_suite( + tmp_path, + "def test_pass(ci_metric):\n ci_metric('Size', 1, 'MB')\n", + "--ci-report=report.json", + "--junitxml=report.xml", + ) + assert result.returncode == 0, result.stdout + result.stderr + assert 'name="ci_metric"' in (tmp_path / "report.xml").read_text() + data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text()) + assert data.metrics[0].value == 1 diff --git a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py index 0eae041f535..9ae7b199e61 100644 --- a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py @@ -4,7 +4,7 @@ import importlib.util import subprocess import sys import zipfile -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping from pathlib import Path from types import MappingProxyType, ModuleType from typing import Final, Protocol, cast @@ -23,16 +23,35 @@ class _CommandRunner(Protocol): ) -> subprocess.CompletedProcess[str]: ... +class _WheelFacts(Protocol): + python_tag: str + abi_tag: str + platform_tag: str + expected_dist_info_directory: str + dist_info_directories: frozenset[str] + dist_info_file_counts: Mapping[str, int] + wheel_metadata_tags: tuple[str, ...] + expanded_filename_tags: frozenset[str] + compressed_size: int + uncompressed_size: int + native_size: int + debug_sections: tuple[str, ...] + static_symbol_table_absent: bool + extension_entry_point_present: bool + native_module_loads: bool + panic_test_hook_absent: bool + duplicated_vocabularies: tuple[str, ...] + unexpected_members: tuple[str, ...] + + class _VerifierModule(Protocol): - main: Callable[ - [ - Sequence[str] | None, - Mapping[str, str] | None, - Callable[[Path], ModuleType | None], - _CommandRunner, - ], - int, - ] + def inspect_wheel( + self, + wheel: Path, + environment: Mapping[str, str] | None = None, + load_native_module: Callable[[Path], ModuleType | None] = ..., + run_command: _CommandRunner = ..., + ) -> _WheelFacts: ... _REPO_ROOT: Final = Path(__file__).resolve().parents[3] @@ -80,21 +99,28 @@ def _write_wheel( return wheel -def _fake_subprocess_run( - command: tuple[str, ...], - *, - check: bool, - capture_output: bool, - text: bool, -) -> subprocess.CompletedProcess[str]: - assert check and capture_output and text - if command == ("rustc", "--version"): - return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") - if "--sections" in command: - return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="") - if "--dyn-syms" in command: - return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") - raise AssertionError(f"unexpected subprocess command: {command}") +_STRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n" +_UNSTRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n[ 2] .debug_info PROGBITS\n[ 3] .symtab SYMTAB\n" + + +def _fake_runner(sections: str = _STRIPPED_SECTIONS) -> _CommandRunner: + def run( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert check and capture_output and text + if command == ("rustc", "--version"): + return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") + if "--sections" in command: + return subprocess.CompletedProcess(command, 0, stdout=sections, stderr="") + if "--dyn-syms" in command: + return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") + raise AssertionError(f"unexpected subprocess command: {command}") + + return run class _NativeModuleWithPanicHook(ModuleType): @@ -102,11 +128,12 @@ class _NativeModuleWithPanicHook(ModuleType): return None -def _run_verifier( +def _inspect( wheel: Path, *, exposes_panic: bool = False, -) -> int: + sections: str = _STRIPPED_SECTIONS, +) -> _WheelFacts: native_module: Final = ( _NativeModuleWithPanicHook("litellm.rust_bridge._native") if exposes_panic @@ -116,33 +143,61 @@ def _run_verifier( def _fake_load_native_module(_: Path) -> ModuleType: return native_module - environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) - return verifier.main( - (str(_MODULE_PATH), str(wheel)), + environment: Final = MappingProxyType({"RELEASE_WHEEL_COMMIT_SHA": "0" * 40}) + return verifier.inspect_wheel( + wheel, environment, _fake_load_native_module, - _fake_subprocess_run, + _fake_runner(sections), ) -def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: +def test_reports_expected_release_wheel_tags(tmp_path: Path) -> None: + native_bytes: Final = b"synthetic native extension" + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, native_bytes=native_bytes) + + facts: Final = _inspect(wheel) + + assert (facts.python_tag, facts.abi_tag, facts.platform_tag) == ("cp310", "abi3", "linux_x86_64") + assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,)) + assert facts.expected_dist_info_directory == _DIST_INFO + assert dict(facts.dist_info_file_counts) == {"METADATA": 1, "RECORD": 1, "WHEEL": 1} + assert frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags + assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags) + assert facts.native_size == len(native_bytes) + assert facts.compressed_size == wheel.stat().st_size + assert facts.uncompressed_size >= facts.native_size > 0 + assert not facts.duplicated_vocabularies + assert not facts.unexpected_members + assert facts.extension_entry_point_present and facts.native_module_loads + assert facts.panic_test_hook_absent + + +def test_reports_debug_sections_and_static_symbol_table(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(wheel) == 0 + facts: Final = _inspect(wheel, sections=_UNSTRIPPED_SECTIONS) + + assert facts.debug_sections == (".debug_",) + assert not facts.static_symbol_table_absent -def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None: +def test_reports_cp312_version_specific_tags(tmp_path: Path) -> None: tag: Final = "cp312-cp312-linux_x86_64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert (facts.python_tag, facts.abi_tag) == ("cp312", "cp312") -def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: +def test_reports_non_linux_platform_tag(tmp_path: Path) -> None: tag: Final = "cp310-abi3-win_amd64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert facts.platform_tag == "win_amd64" @pytest.mark.parametrize( @@ -150,16 +205,18 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: (None, ("cp312-cp312-linux_x86_64",)), ids=("missing", "mismatched"), ) -def test_rejects_missing_or_mismatched_wheel_metadata_tag( +def test_reports_missing_or_mismatched_wheel_metadata_tag( tmp_path: Path, metadata_tags: tuple[str, ...] | None, ) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert frozenset(facts.wheel_metadata_tags) != facts.expanded_filename_tags -def test_rejects_wheel_metadata_from_wrong_dist_info_directory( +def test_reports_wheel_metadata_from_wrong_dist_info_directory( tmp_path: Path, ) -> None: wheel: Final = _write_wheel( @@ -168,20 +225,27 @@ def test_rejects_wheel_metadata_from_wrong_dist_info_directory( dist_info="decoy-1.0.0.dist-info", ) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert facts.dist_info_directories == frozenset(("decoy-1.0.0.dist-info",)) + assert facts.dist_info_directories != frozenset((facts.expected_dist_info_directory,)) + assert dict(facts.dist_info_file_counts) == {"METADATA": 0, "RECORD": 0, "WHEEL": 0} -def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: +def test_reports_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: wheel: Final = _write_wheel( tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), ) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert facts.wheel_metadata_tags == (_EXPECTED_TAG, _EXPECTED_TAG) + assert len(facts.wheel_metadata_tags) != len(facts.expanded_filename_tags) -def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: +def test_reports_duplicate_wheel_metadata_file(tmp_path: Path) -> None: with pytest.warns(UserWarning, match="Duplicate name"): wheel: Final = _write_wheel( tmp_path, @@ -189,13 +253,17 @@ def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: duplicate_wheel=True, ) - assert _run_verifier(wheel) == 1 + facts: Final = _inspect(wheel) + + assert facts.dist_info_file_counts["WHEEL"] == 2 -def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: +def test_reports_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(wheel, exposes_panic=True) == 1 + facts: Final = _inspect(wheel, exposes_panic=True) + + assert not facts.panic_test_hook_absent @pytest.mark.parametrize("embedded", (False, True)) @@ -209,4 +277,6 @@ def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: with zipfile.ZipFile(wheel, "a") as archive: archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) - assert _run_verifier(wheel) == (1 if embedded else 0) + facts: Final = _inspect(wheel) + + assert bool(facts.duplicated_vocabularies) == embedded diff --git a/tests/unit/test_unit_shard_missing_paths.py b/tests/unit/test_unit_shard_missing_paths.py index 725551e19d3..60cc1390833 100644 --- a/tests/unit/test_unit_shard_missing_paths.py +++ b/tests/unit/test_unit_shard_missing_paths.py @@ -1,3 +1,4 @@ +import json import os import subprocess import sys @@ -28,7 +29,9 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl shim_dir.mkdir() (shim_dir / "uv").write_text(_UV_SHIM) (shim_dir / "uv").chmod(0o755) - (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n") + (tmp_path / "pyproject.toml").write_text( + f"[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\npythonpath = [{json.dumps(str(_REPO_ROOT / 'scripts'))}]\n" + ) return subprocess.run( ("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()), cwd=tmp_path, diff --git a/tests/wheel/AGENTS.md b/tests/wheel/AGENTS.md new file mode 100644 index 00000000000..fe8dbb52523 --- /dev/null +++ b/tests/wheel/AGENTS.md @@ -0,0 +1,7 @@ +# Release wheel checks + +- Run against a release-built wheel passed with `--wheel` +- Only run in the `rust-wheel` job, never select from a unit shard +- May invoke `readelf` and load the compiled native extension +- Keep facts in `.github/scripts/verify_linux_native_wheel.py` and assertions here +- Record build metrics with `ci_metric`, never write a separate summary diff --git a/tests/wheel/conftest.py b/tests/wheel/conftest.py new file mode 100644 index 00000000000..bb31ba69ec2 --- /dev/null +++ b/tests/wheel/conftest.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Final + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".github" / "scripts")) # test-quality-ok: the wheel inspector lives in .github/scripts, outside any importable package + +from verify_linux_native_wheel import WheelFacts, WheelInspectionError, inspect_wheel # noqa: E402 # CI helpers live outside Python packages + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption("--wheel", help="Release-built native wheel to inspect") + + +@pytest.fixture(scope="session") +def facts(request: pytest.FixtureRequest) -> WheelFacts: + wheel: Final[object] = request.config.getoption("wheel") + if not isinstance(wheel, str): + pytest.fail("--wheel is required for release wheel checks") + inspected: Final = inspect_wheel(Path(wheel)) + if isinstance(inspected, WheelInspectionError): + pytest.fail(inspected.message) + return inspected diff --git a/tests/wheel/test_linux_native_wheel.py b/tests/wheel/test_linux_native_wheel.py new file mode 100644 index 00000000000..1a70595b717 --- /dev/null +++ b/tests/wheel/test_linux_native_wheel.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from pytest_ci_report import CIMetric +from verify_linux_native_wheel import ( + EXPECTED_ABI_TAG, + EXPECTED_PLATFORM_TAG, + EXPECTED_PYTHON_TAG, + WheelFacts, +) + + +def test_python_tag_is_cp310(facts: WheelFacts) -> None: + assert facts.python_tag == EXPECTED_PYTHON_TAG, f"unexpected Python tag: {facts.python_tag}" + + +def test_abi_tag_is_abi3(facts: WheelFacts) -> None: + assert facts.abi_tag == EXPECTED_ABI_TAG, f"unexpected ABI tag: {facts.abi_tag}" + + +def test_platform_tag_is_linux_x86_64(facts: WheelFacts) -> None: + assert facts.platform_tag == EXPECTED_PLATFORM_TAG, f"unexpected platform tag: {facts.platform_tag}" + + +def test_dist_info_directory_matches_filename(facts: WheelFacts) -> None: + assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,)), ( + f"unexpected dist-info directories: {facts.dist_info_directories}" + ) + + +def test_required_dist_info_files_are_present_once(facts: WheelFacts) -> None: + assert all(count == 1 for count in facts.dist_info_file_counts.values()), ( + f"required dist-info file counts are invalid: {facts.dist_info_file_counts}" + ) + + +def test_metadata_tags_match_filename(facts: WheelFacts) -> None: + assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags) and ( + frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags + ), f"WHEEL tags do not match filename: {facts.wheel_metadata_tags}" + + +def test_debug_sections_are_absent(facts: WheelFacts) -> None: + assert not facts.debug_sections, f"native extension contains debug sections: {facts.debug_sections}" + + +def test_static_symbol_table_is_absent(facts: WheelFacts) -> None: + assert facts.static_symbol_table_absent, "native extension contains a static symbol table" + + +def test_python_extension_entry_point_is_present(facts: WheelFacts) -> None: + assert facts.extension_entry_point_present, "native extension does not export PyInit__native" + + +def test_native_module_loads(facts: WheelFacts) -> None: + assert facts.native_module_loads, "native extension cannot be loaded" + + +def test_production_module_omits_panic_hook(facts: WheelFacts) -> None: + assert facts.panic_test_hook_absent, "production native extension exposes _panic_for_test" + + +def test_native_extension_does_not_exceed_48_mb(facts: WheelFacts) -> None: + assert facts.native_size <= 48_000_000, ( + f"native extension exceeds 48 MB: {facts.native_size / 1_000_000:.2f} MB" + ) + + +def test_tokenizer_vocabularies_are_not_duplicated(facts: WheelFacts) -> None: + assert not facts.duplicated_vocabularies, ( + f"tokenizer vocabularies are duplicated in the native extension: {facts.duplicated_vocabularies}" + ) + + +def test_wheel_contents_are_valid(facts: WheelFacts) -> None: + assert not facts.unexpected_members, f"wheel contains unexpected build artifacts: {facts.unexpected_members}" + + +def test_build_report(facts: WheelFacts, ci_metric: CIMetric) -> None: + assert 0 < facts.native_size <= facts.uncompressed_size + ci_metric("Compressed wheel", round(facts.compressed_size / 1_000_000, 2), "MB") + ci_metric("Uncompressed wheel", round(facts.uncompressed_size / 1_000_000, 2), "MB") + ci_metric("Native extension", round(facts.native_size / 1_000_000, 2), "MB") + ci_metric("Native share", round(facts.native_size / facts.uncompressed_size * 100, 2), "%") + ci_metric("Commit", facts.commit_sha) + ci_metric("Platform", facts.platform_tag) + ci_metric("Python ABI", f"{facts.python_tag}-{facts.abi_tag}") + ci_metric("Rust compiler", facts.rustc_version) + ci_metric("Maturin", facts.maturin_version) + ci_metric("Cargo profile", "release")