From f0ddb5045c62043ea3f55045a4d86712651d32b8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 21 Sep 2026 01:18:19 +0000 Subject: [PATCH] ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../scripts/verify_installed_wheel_imports.py | 122 ------------------ .github/workflows/codspeed.yml | 10 -- .github/workflows/test-rust.yml | 6 +- Makefile | 6 +- tests/benchmarks/conftest.py | 42 ++---- .../test_verify_installed_wheel_imports.py | 93 ------------- 6 files changed, 11 insertions(+), 268 deletions(-) delete mode 100644 .github/scripts/verify_installed_wheel_imports.py delete mode 100644 tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py diff --git a/.github/scripts/verify_installed_wheel_imports.py b/.github/scripts/verify_installed_wheel_imports.py deleted file mode 100644 index 7775dd333f0..00000000000 --- a/.github/scripts/verify_installed_wheel_imports.py +++ /dev/null @@ -1,122 +0,0 @@ -from __future__ import annotations - -import importlib.metadata -import json -import sys -from collections.abc import Mapping, Sequence -from pathlib import Path -from types import MappingProxyType -from typing import Final, cast - - -def _direct_url_failures(direct_url: str | None) -> tuple[str, ...]: - if direct_url is None: - return () - try: - direct_url_data: Final = cast(object, json.loads(direct_url)) - except json.JSONDecodeError: - return ("litellm distribution direct_url.json is invalid JSON",) - if not isinstance(direct_url_data, Mapping): - return () - direct_url_mapping: Final = cast(Mapping[str, object], direct_url_data) - dir_info: Final = direct_url_mapping.get("dir_info") - if not isinstance(dir_info, Mapping): - return () - dir_info_mapping: Final = cast(Mapping[str, object], dir_info) - return ("litellm distribution is editable",) if dir_info_mapping.get("editable") is True else () - - -def _native_bridge_available() -> bool: - try: - from litellm.rust_bridge import native_bridge_available - except Exception: - return False - return native_bridge_available() - - -def provenance_failures( - *, - prefix: Path, - checkout: Path, - module_files: Mapping[str, Path], - direct_url: str | None, - native_available: bool, -) -> tuple[str, ...]: - resolved_prefix: Final = prefix.resolve() - resolved_checkout: Final = checkout.resolve() - resolved_module_files: Final = tuple( - (module_name, module_path.resolve()) for module_name, module_path in module_files.items() - ) - prefix_failures: Final = tuple( - f"{module_name} is not under sys.prefix: {module_path}" - for module_name, module_path in resolved_module_files - if not module_path.is_relative_to(resolved_prefix) - ) - checkout_failures: Final = tuple( - f"{module_name} is under the checkout: {module_path}" - for module_name, module_path in resolved_module_files - if module_path.is_relative_to(resolved_checkout) - ) - direct_url_failures: Final = _direct_url_failures(direct_url) - native_failures: Final = ("litellm.rust_bridge.native_bridge_available() is false",) if not native_available else () - return prefix_failures + checkout_failures + direct_url_failures + native_failures - - -def main(argv: Sequence[str] | None = None) -> int: - arguments: Final = tuple(sys.argv if argv is None else argv) - if len(arguments) != 2: - sys.stderr.write(f"usage: {Path(arguments[0]).name} CHECKOUT\n") - return 2 - - checkout: Final = Path(arguments[1]) - try: - import litellm - except Exception as error: - sys.stderr.write(f"litellm import failed: {error}\n") - return 1 - - try: - import litellm.rust_bridge._native as native - except Exception as error: - sys.stderr.write(f"litellm.rust_bridge._native import failed: {error}\n") - return 1 - - imported_modules: Final = ( - ("litellm", litellm), - ("litellm.rust_bridge._native", native), - ) - missing_module_files: Final = tuple( - f"{module_name} has no __file__" for module_name, module in imported_modules if module.__file__ is None - ) - module_files: Final = MappingProxyType( - { - module_name: Path(module.__file__).resolve() - for module_name, module in imported_modules - if module.__file__ is not None - } - ) - try: - direct_url: Final = importlib.metadata.distribution("litellm").read_text("direct_url.json") - except Exception as error: - sys.stderr.write(f"litellm distribution lookup failed: {error}\n") - return 1 - - native_available: Final = _native_bridge_available() - - failures: Final = missing_module_files + provenance_failures( - prefix=Path(sys.prefix), - checkout=checkout, - module_files=module_files, - direct_url=direct_url, - native_available=native_available, - ) - if failures: - sys.stderr.write("".join(f"{failure}\n" for failure in failures)) - return 1 - - sys.stdout.write("".join(f"{module_name}: {module_path}\n" for module_name, module_path in module_files.items())) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 692445be7f6..ec7e211faa1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,8 +13,6 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" - - ".github/scripts/verify_installed_wheel_imports.py" - - ".github/scripts/verify_linux_native_wheel.py" - ".github/scripts/uv_sync_with_retries.sh" pull_request: branches: @@ -28,8 +26,6 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" - - ".github/scripts/verify_installed_wheel_imports.py" - - ".github/scripts/verify_linux_native_wheel.py" - ".github/scripts/uv_sync_with_retries.sh" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data @@ -74,16 +70,10 @@ jobs: - name: Build the release wheel run: uv build --wheel --out-dir dist - - name: Verify the release wheel - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - env: - RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Install the wheel into the benchmark environment run: | UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12 uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl - (cd "${RUNNER_TEMP}" && "${RUNNER_TEMP}/benchmark-venv/bin/python" -I "${GITHUB_WORKSPACE}/.github/scripts/verify_installed_wheel_imports.py" "${GITHUB_WORKSPACE}") - name: Collect benchmarks env: diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 4d9f7140f70..70d828b5c8b 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -24,9 +24,7 @@ on: - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - - ".github/scripts/verify_installed_wheel_imports.py" - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - - "tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -56,9 +54,7 @@ on: - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - - ".github/scripts/verify_installed_wheel_imports.py" - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - - "tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py" - ".github/workflows/test-rust.yml" permissions: @@ -163,7 +159,7 @@ jobs: - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl - name: Run pytest tests/test_litellm_rust with the compiled extension - run: make test-rust-extension WHEEL="$(echo dist/*.whl)" + run: make test-rust-extension - run: >- uv build --wheel --out-dir panic-dist diff --git a/Makefile b/Makefile index 27bb5abe540..0e9d2bbf82c 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ help: @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" - @echo " make test-rust-extension - Build the Rust extension and run its public Python tests (WHEEL=path/to.whl reuses a prebuilt wheel)" + @echo " make test-rust-extension - Build the Rust extension and run its public Python tests" @echo "" @echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide" @echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine." @@ -294,13 +294,11 @@ pre-commit: test-rust-extension: @temporary=$$(mktemp -d) && \ trap 'rm -rf "$$temporary"' EXIT HUP INT TERM && \ - mkdir -p "$$temporary/wheels" && \ - if [ -n "$(WHEEL)" ]; then cp $(WHEEL) "$$temporary/wheels/"; else $(UV) build --python 3.12 --wheel --out-dir "$$temporary/wheels"; fi && \ + $(UV) build --python 3.12 --wheel --out-dir "$$temporary/wheels" && \ set -- "$$temporary"/wheels/*.whl && \ [ "$$#" -eq 1 ] && \ UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \ $(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \ - (cd "$$temporary" && "$$temporary/venv/bin/python" -I "$(CURDIR)/.github/scripts/verify_installed_wheel_imports.py" "$(CURDIR)") && \ "$$temporary/venv/bin/python" -I -m mypy.stubtest \ --mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \ litellm.rust_bridge._native && \ diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 659a27ce392..309ecab9991 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -8,14 +8,12 @@ flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. """ -import importlib.metadata -import json import os import sys -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator from concurrent.futures import Future from pathlib import Path -from typing import Final, ParamSpec, TypeVar, cast +from typing import ParamSpec, TypeVar import pytest @@ -32,36 +30,12 @@ def pytest_configure(config: pytest.Config) -> None: import litellm import litellm.rust_bridge._native as native - prefix: Final = Path(sys.prefix).resolve() - module_paths: Final = ( - ("litellm", Path(litellm.__file__).resolve()), - ("litellm.rust_bridge._native", Path(native.__file__).resolve()), - ) - for module_name, module_path in module_paths: - sys.stdout.write(f"{module_name}: {module_path}\n") - - path_failures: Final = tuple( - f"{module_name} is not under sys.prefix: {module_path}" - for module_name, module_path in module_paths - if not module_path.is_relative_to(prefix) - ) - if path_failures: - raise pytest.UsageError("; ".join(path_failures)) - - direct_url: Final = importlib.metadata.distribution("litellm").read_text("direct_url.json") - if direct_url is None: - return - - try: - direct_url_data: Final = cast(object, json.loads(direct_url)) - except json.JSONDecodeError as error: - raise pytest.UsageError("litellm distribution direct_url.json is invalid JSON") from error - - if isinstance(direct_url_data, Mapping): - direct_url_mapping: Final = cast(Mapping[str, object], direct_url_data) - dir_info: Final = direct_url_mapping.get("dir_info") - if isinstance(dir_info, Mapping) and cast(Mapping[str, object], dir_info).get("editable") is True: - raise pytest.UsageError("litellm distribution is editable") + prefix = Path(sys.prefix).resolve() + for name, module_file in (("litellm", litellm.__file__), ("litellm.rust_bridge._native", native.__file__)): + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise pytest.UsageError(f"{name} resolved outside the benchmark environment: {path}") + print(f"{name}: {path}") # noqa: T201 # provenance evidence must be visible in CI logs def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: diff --git a/tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py b/tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py deleted file mode 100644 index 729f56ce993..00000000000 --- a/tests/test_litellm/rust_bridge/test_verify_installed_wheel_imports.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -from collections.abc import Mapping -from pathlib import Path -from typing import Final, Protocol, cast - - -class _VerifierModule(Protocol): - def provenance_failures( - self, - *, - prefix: Path, - checkout: Path, - module_files: Mapping[str, Path], - direct_url: str | None, - native_available: bool, - ) -> tuple[str, ...]: ... - - -_REPO_ROOT: Final = Path(__file__).resolve().parents[3] -_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_installed_wheel_imports.py" -_SPEC: Final = importlib.util.spec_from_file_location("verify_installed_wheel_imports", _MODULE_PATH) -assert _SPEC is not None and _SPEC.loader is not None -_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = _LOADED_VERIFIER -_SPEC.loader.exec_module(_LOADED_VERIFIER) -verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) - - -def _failures( - *, - prefix: Path, - checkout: Path, - module_files: Mapping[str, Path], - direct_url: str | None = '{"dir_info": {"editable": false}}', - native_available: bool = True, -) -> tuple[str, ...]: - return verifier.provenance_failures( - prefix=prefix, - checkout=checkout, - module_files=module_files, - direct_url=direct_url, - native_available=native_available, - ) - - -def test_accepts_installed_non_editable_modules(tmp_path: Path) -> None: - prefix: Final = tmp_path / "venv" / "lib" / "python3.12" / "site-packages" - checkout: Final = tmp_path / "checkout" - module_files: Final = { - "litellm": prefix / "litellm" / "__init__.py", - "litellm.rust_bridge._native": prefix / "litellm" / "rust_bridge" / "_native.abi3.so", - } - - assert _failures(prefix=prefix, checkout=checkout, module_files=module_files) == () - - -def test_rejects_module_under_checkout(tmp_path: Path) -> None: - prefix: Final = tmp_path - checkout: Final = tmp_path / "checkout" - module_files: Final = {"litellm": checkout / "litellm" / "__init__.py"} - - assert _failures(prefix=prefix, checkout=checkout, module_files=module_files) == ( - f"litellm is under the checkout: {(checkout / 'litellm' / '__init__.py').resolve()}", - ) - - -def test_rejects_editable_direct_url(tmp_path: Path) -> None: - prefix: Final = tmp_path / "site-packages" - checkout: Final = tmp_path / "checkout" - module_files: Final = {"litellm": prefix / "litellm" / "__init__.py"} - - assert _failures( - prefix=prefix, - checkout=checkout, - module_files=module_files, - direct_url='{"dir_info": {"editable": true}}', - ) == ("litellm distribution is editable",) - - -def test_rejects_unavailable_native_bridge(tmp_path: Path) -> None: - prefix: Final = tmp_path / "site-packages" - checkout: Final = tmp_path / "checkout" - module_files: Final = {"litellm": prefix / "litellm" / "__init__.py"} - - assert _failures( - prefix=prefix, - checkout=checkout, - module_files=module_files, - native_available=False, - ) == ("litellm.rust_bridge.native_bridge_available() is false",)