From c33a079c5ffd1d27a989003aa1537266d02ef61b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:42:48 -0700 Subject: [PATCH] test(build): validate release wheel contracts --- .github/scripts/smoke_test_native_wheel.py | 68 ++++++++ .github/scripts/verify_linux_native_wheel.py | 161 +++++++++++++++++++ .github/workflows/test-rust.yml | 111 +++++++++++++ litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/lib.rs | 8 + 5 files changed, 349 insertions(+) create mode 100644 .github/scripts/smoke_test_native_wheel.py create mode 100644 .github/scripts/verify_linux_native_wheel.py diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py new file mode 100644 index 00000000000..577bb32fcf0 --- /dev/null +++ b/.github/scripts/smoke_test_native_wheel.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Final + +CHILD_SCRIPT: Final = """ +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +native_path = Path(sys.argv[1]) +spec = spec_from_file_location("litellm.rust_bridge._native", native_path) +if spec is None or spec.loader is None: + raise RuntimeError("cannot create native extension import specification") +module = module_from_spec(spec) +spec.loader.exec_module(module) + +before = module.gil_stats() +if not isinstance(before.get("releases"), int): + raise AssertionError(f"unexpected gil_stats result: {before!r}") + +try: + module._panic_for_test() +except BaseException as error: + if type(error).__name__ != "PanicException": + raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error +else: + raise AssertionError("Rust panic returned without raising") + +after = module.gil_stats() +if not isinstance(after.get("releases"), int): + raise AssertionError(f"native module unusable after panic: {after!r}") +""" + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name + native_path.write_bytes(archive.read(native_members[0])) + result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + + if result.returncode != 0: + sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py new file mode 100644 index 00000000000..1de98ae0168 --- /dev/null +++ b/.github/scripts/verify_linux_native_wheel.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path, PurePosixPath +from typing import Final + + +def _loads_native_module(native_path: Path) -> bool: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + return False + try: + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + except Exception as error: + sys.stderr.write(f"native module load failed: {error}\n") + return False + return True + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with zipfile.ZipFile(wheel) as archive: + wheel_members: Final = archive.infolist() + native_members: Final = tuple( + member + for member in wheel_members + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + unexpected_members: Final = tuple( + member.filename + for member in wheel_members + if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock")) + or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts) + ) + native_member: Final = native_members[0] + uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) + native_path: Final = wheel.parent / "native" / Path(native_member.filename).name + native_path.parent.mkdir(parents=True, exist_ok=True) + native_path.write_bytes(archive.read(native_member)) + + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) + rustc_version: Final = subprocess.run( + ("rustc", "--version"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) + if maturin_match is None: + sys.stderr.write("build-system does not pin an exact Maturin version\n") + return 1 + + maturin_version: Final = maturin_match.group(1) + native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 + size_report: Final = "\n".join( + ( + "## Native wheel build report", + "", + "| Build | Value |", + "| --- | --- |", + f"| Commit | `{commit_sha}` |", + f"| Platform | `{platform_tag}` |", + f"| Python ABI | `{python_tag}-{abi_tag}` |", + f"| Rust compiler | `{rustc_version}` |", + f"| Maturin | `{maturin_version}` |", + "| Cargo profile | `release` |", + "", + "| Artifact | Size |", + "| --- | ---: |", + f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", + f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", + f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| Native share | {native_percentage:.2f}% |", + "", + ) + ) + summary_path: Final = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path is None: + sys.stdout.write(size_report) + else: + Path(summary_path).write_text(size_report) + + sections: Final = subprocess.run( + ("readelf", "--sections", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) + debug_sections_absent: Final = not debug_sections + static_symbol_table_absent: Final = ".symtab" not in sections + + dynamic_symbols: Final = subprocess.run( + ("readelf", "--dyn-syms", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols + native_module_loads: Final = _loads_native_module(native_path) + native_size_limit: Final = 20_000_000 + native_size_within_limit: Final = native_member.file_size <= native_size_limit + validations: Final = ( + ("Debug sections are absent", debug_sections_absent), + ("Static symbol table is absent", static_symbol_table_absent), + ("Python extension entry point is present", extension_entry_point_present), + ("Native module loads", native_module_loads), + ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Wheel contents are valid", not unexpected_members), + ) + + verified_report: Final = size_report + "\n".join( + ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") + + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + ("",) + ) + report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") + if report_path is not None: + Path(report_path).write_text(size_report) + if summary_path is not None: + Path(summary_path).write_text(verified_report) + + if debug_sections: + sys.stderr.write(f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}\n") + if not static_symbol_table_absent: + sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") + if not extension_entry_point_present: + sys.stderr.write("native extension does not export PyInit__native\n") + if not native_size_within_limit: + sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") + if unexpected_members: + sys.stderr.write(f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}\n") + + return 0 if all(passed for _, passed in validations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 21e1bcb90c6..b05ddb81740 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,6 +4,10 @@ on: push: paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -13,6 +17,10 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -69,3 +77,106 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + + release-wheel: + name: release wheel + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Cache release build + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-release- + + - name: Build release wheel + run: uv build --wheel --out-dir dist + + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RELEASE_WHEEL_REPORT: dist/release-wheel-report.md + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + + - name: Build panic contract wheel + run: >- + uv build --wheel --out-dir panic-dist + --config-setting "maturin.build-args=--features panic-test" + + - name: Smoke-test native panic unwinding + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + - name: Report release wheel size on PR + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + REPORT_PATH: dist/release-wheel-report.md + with: + script: | + const fs = require("fs"); + const marker = process.env.COMMENT_MARKER; + const report = fs.readFileSync(process.env.REPORT_PATH, "utf8"); + const body = `${marker}\n${report}`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Upload release wheel + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-release-wheel-linux-x86_64 + path: dist/*.whl + if-no-files-found: error diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index d461a483ae0..b1fdfd7677a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] +panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..18e0b05cbb5 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -614,6 +614,12 @@ fn gil_stats(py: Python<'_>) -> PyResult> { Ok(stats.into_any().unbind()) } +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + #[pymodule] fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); @@ -630,5 +636,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(achat_completions, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; Ok(()) }