diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 4c4cb399dce..9df01c07a72 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -7,14 +7,62 @@ import subprocess import sys import zipfile from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from itertools import product from pathlib import Path, PurePosixPath from types import MappingProxyType, ModuleType from typing import Final, Protocol -EXPECTED_PYTHON_TAG: Final = "cp310" -EXPECTED_ABI_TAG: Final = "abi3" EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" +REPO_ROOT: Final = Path(__file__).parents[2] + + +@dataclass(frozen=True, slots=True) +class WheelProfile: + distribution: str + python_tag: str + abi_tag: str + project_directory: Path + label: str + report_title: str + size_limit: int + is_payload: Callable[[str], bool] + is_extension: bool + + +PROFILES: Final = MappingProxyType( + { + "extension": WheelProfile( + distribution="litellm", + python_tag="cp310", + abi_tag="abi3", + project_directory=REPO_ROOT, + label="Native extension", + report_title="Native wheel build report", + size_limit=48_000_000, + is_payload=lambda name: name.startswith("litellm/rust_bridge/_native.") and name.endswith(".so"), + is_extension=True, + ), + "gateway": WheelProfile( + distribution="litellm_gateway", + python_tag="py3", + abi_tag="none", + project_directory=REPO_ROOT / "litellm-rust" / "crates" / "gateway", + label="Gateway binary", + report_title="Gateway wheel build report", + size_limit=37_000_000, + is_payload=lambda name: name.endswith(".data/scripts/litellm-gateway"), + is_extension=False, + ), + } +) + + +@dataclass(frozen=True, slots=True) +class Validation: + label: str + passed: bool + failure: str class CommandRunner(Protocol): @@ -65,6 +113,57 @@ def _load_native_module(native_path: Path) -> ModuleType | None: return native_module +def _extension_validations( + wheel: Path, + native_path: Path, + native_bytes: bytes, + load_native_module: Callable[[Path], ModuleType | None], + run_command: CommandRunner, +) -> tuple[Validation, ...]: + with zipfile.ZipFile(wheel) as archive: + duplicated_vocabularies: Final = tuple( + member.filename + for member in archive.infolist() + if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") + and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) + and member.file_size > 0 + and archive.read(member) in native_bytes + ) + dynamic_symbols: Final = run_command( + ("readelf", "--dyn-syms", "--wide", str(native_path)), + check=True, + capture_output=True, + text=True, + ).stdout + native_module: Final = load_native_module(native_path) + return ( + Validation( + "Tokenizer vocabularies are not duplicated in the native extension", + not duplicated_vocabularies, + f"native extension duplicates tokenizer vocabularies: {', '.join(duplicated_vocabularies)}", + ), + Validation( + "Python extension entry point is present", + "PyInit__native" in dynamic_symbols, + "native extension does not export PyInit__native", + ), + Validation("Native module loads", native_module is not None, "native module did not load"), + Validation( + "Production module omits the panic test hook", + native_module is None or not hasattr(native_module, "_panic_for_test"), + "production native module exposes _panic_for_test", + ), + ) + + +def _write_report(report: str, summary_path: str | None) -> None: + if summary_path is None: + sys.stdout.write(report + "\n") + return + with Path(summary_path).open("a") as summary: + summary.write(report + "\n") + + def main( argv: Sequence[str] | None = None, environment: Mapping[str, str] | None = None, @@ -73,18 +172,19 @@ def main( ) -> int: arguments: Final = tuple(sys.argv if argv is None else argv) resolved_environment: Final = os.environ if environment is None else environment - if len(arguments) != 2: - sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") + if len(arguments) != 3 or arguments[1] not in PROFILES: + sys.stderr.write("usage: verify_linux_native_wheel.py {extension|gateway} WHEEL\n") return 2 - wheel: Final = Path(arguments[1]) + profile: Final = PROFILES[arguments[1]] + wheel: Final = Path(arguments[2]) wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) if len(wheel_tags) != 4: sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") return 1 wheel_identity: Final = wheel_tags[0].split("-") - if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: + if len(wheel_identity) != 2 or wheel_identity[0] != profile.distribution or not wheel_identity[1]: sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") return 1 @@ -115,13 +215,9 @@ def main( member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" ) wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members) - native_members: Final = tuple( - member - for member in wheel_members - if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") - ) + native_members: Final = tuple(member for member in wheel_members if profile.is_payload(member.filename)) if len(native_members) != 1: - sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + sys.stderr.write(f"expected one {profile.label.lower()}, found {len(native_members)}\n") return 1 unexpected_members: Final = tuple( @@ -136,14 +232,6 @@ def main( native_path.parent.mkdir(parents=True, exist_ok=True) native_bytes: Final = archive.read(native_member) native_path.write_bytes(native_bytes) - duplicated_vocabularies: Final = tuple( - member.filename - for member in wheel_members - if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") - and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) - and member.file_size > 0 - and archive.read(member) in native_bytes - ) wheel_metadata_tags_match: Final = ( len(wheel_metadata_tags) == len(expanded_filename_tags) @@ -158,7 +246,7 @@ def main( capture_output=True, text=True, ).stdout.strip() - pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + pyproject: Final = (profile.project_directory / "pyproject.toml").read_text() maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) if maturin_match is None: sys.stderr.write("build-system does not pin an exact Maturin version\n") @@ -168,7 +256,7 @@ def main( native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 size_report: Final = "\n".join( ( - "## Native wheel build report", + f"## {profile.report_title}", "", "| Build | Value |", "| --- | --- |", @@ -183,16 +271,13 @@ def main( "| --- | ---: |", f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", - f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| {profile.label} | {native_member.file_size / 1_000_000:.2f} MB |", f"| Native share | {native_percentage:.2f}% |", "", ) ) summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") - if summary_path is None: - sys.stdout.write(size_report) - else: - Path(summary_path).write_text(size_report) + _write_report(size_report, summary_path) sections: Final = run_command( ("readelf", "--sections", "--wide", str(native_path)), @@ -201,92 +286,77 @@ def main( text=True, ).stdout debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) - debug_sections_absent: Final = not debug_sections - static_symbol_table_absent: Final = ".symtab" not in sections - - dynamic_symbols: Final = run_command( - ("readelf", "--dyn-syms", "--wide", str(native_path)), - check=True, - capture_output=True, - text=True, - ).stdout - extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols - native_module: Final = load_native_module(native_path) - native_module_loads: Final = native_module is not None - panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 48_000_000 - native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( - (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), - (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), - (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), - ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), - ( + Validation( + f"Python tag is {profile.python_tag}", + python_tag == profile.python_tag, + f"unexpected Python tag: expected {profile.python_tag}, found {python_tag}", + ), + Validation( + f"ABI tag is {profile.abi_tag}", + abi_tag == profile.abi_tag, + f"unexpected ABI tag: expected {profile.abi_tag}, found {abi_tag}", + ), + Validation( + f"Platform tag is {EXPECTED_PLATFORM_TAG}", + platform_tag == EXPECTED_PLATFORM_TAG, + f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", + ), + Validation( + "Wheel dist-info directory matches the filename", + dist_info_directories == expected_dist_info_directories, + f"unexpected dist-info directories: expected {expected_dist_info_directory}, " + f"found {', '.join(sorted(dist_info_directories))}", + ), + Validation( "Required dist-info files are present exactly once", all(count == 1 for count in dist_info_file_counts.values()), + f"required dist-info file counts are invalid: {dist_info_file_counts}", ), - ("Wheel metadata tags match the filename", wheel_metadata_tags_match), - ("Debug sections are absent", debug_sections_absent), - ("Static symbol table is absent", static_symbol_table_absent), - ("Python extension entry point is present", extension_entry_point_present), - ("Native module loads", native_module_loads), - ("Production module omits the panic test hook", panic_test_hook_absent), - (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), - ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), - ("Wheel contents are valid", not unexpected_members), + Validation( + "Wheel metadata tags match the filename", + wheel_metadata_tags_match, + f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " + f"found {', '.join(sorted(wheel_metadata_tags))}", + ), + Validation( + "Debug sections are absent", + not debug_sections, + f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}", + ), + Validation( + "Static symbol table is absent", + ".symtab" not in sections, + f"{native_member.filename} contains a static symbol table", + ), + Validation( + f"{profile.label} does not exceed {profile.size_limit / 1_000_000:.0f} MB", + native_member.file_size <= profile.size_limit, + f"{profile.label.lower()} exceeds {profile.size_limit / 1_000_000:.0f} MB: " + f"{native_member.file_size / 1_000_000:.2f} MB", + ), + Validation( + "Wheel contents are valid", + not unexpected_members, + f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}", + ), + ) + ( + _extension_validations(wheel, native_path, native_bytes, load_native_module, run_command) + if profile.is_extension + else () ) - verified_report: Final = size_report + "\n".join( + verified_report: Final = "\n".join( ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") - + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + tuple(f"| {row.label} | Yes | {'O' if row.passed else 'X'} |" for row in validations) + ("",) ) - if summary_path is not None: - Path(summary_path).write_text(verified_report) + _write_report(verified_report, summary_path) - invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) - validation_errors: Final = tuple( - message - for failed, message in ( - (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), - (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), - (not extension_entry_point_present, "native extension does not export PyInit__native"), - ( - python_tag != EXPECTED_PYTHON_TAG, - f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", - ), - (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), - ( - platform_tag != EXPECTED_PLATFORM_TAG, - f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", - ), - ( - dist_info_directories != expected_dist_info_directories, - f"unexpected dist-info directories: expected {expected_dist_info_directory}, " - f"found {', '.join(sorted(dist_info_directories))}", - ), - (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), - ( - not invalid_dist_info_files and not wheel_metadata_tags_match, - f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " - f"found {', '.join(sorted(wheel_metadata_tags))}", - ), - ( - native_module is not None and not panic_test_hook_absent, - "production native module exposes _panic_for_test", - ), - ( - not native_size_within_limit, - f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: " - f"{native_member.file_size / 1_000_000:.2f} MB", - ), - (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), - ) - if failed - ) + validation_errors: Final = tuple(row.failure for row in validations if not row.passed) sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) - return 0 if all(passed for _, passed in validations) else 1 + return 0 if not validation_errors else 1 if __name__ == "__main__": diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index b9f8d7e7dfb..15fcab3e3bb 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -171,7 +171,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy + uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy --no-install-package litellm-gateway uv run --no-sync python scripts/openapi_docs_gate.py --base "$GATE_BASE_SHA" - name: Print OpenAI version @@ -412,7 +412,7 @@ jobs: - name: Install backend dependencies if: steps.changes.outputs.relevant == 'true' - run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --no-install-package litellm-gateway - name: Cache Prisma binaries if: steps.changes.outputs.relevant == 'true' @@ -516,7 +516,7 @@ jobs: uses: ./.github/actions/cache-cargo-build - name: Install dependencies - run: uv sync --frozen --all-groups --all-extras + run: uv sync --frozen --all-groups --all-extras --no-install-package litellm-gateway - name: check_licenses run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index f3f09523491..7519039507f 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -52,6 +52,9 @@ jobs: with: version: "0.10.9" + - name: Set up Rust before concurrent package builds + run: rustup toolchain install --no-self-update + - name: Install dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }} @@ -100,7 +103,7 @@ jobs: all-extras: name: all-extras-py${{ matrix.python-version }} runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -123,6 +126,9 @@ jobs: with: version: "0.10.9" + - name: Set up Rust before concurrent package builds + run: rustup toolchain install --no-self-update + - name: Install every extra and dependency group run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }} diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 636a5031e5f..01f45dae96f 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -189,7 +189,7 @@ jobs: - run: uv build --wheel --out-dir dist - - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + - run: python .github/scripts/verify_linux_native_wheel.py extension dist/*.whl env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -203,3 +203,45 @@ jobs: --config-setting "maturin.build-args=--features panic-test,extension-module" - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + gateway-wheel: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - run: rustup toolchain install --no-self-update + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: litellm-rust + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + + - run: uv build --package litellm-gateway --wheel --out-dir dist + + - run: python .github/scripts/verify_linux_native_wheel.py gateway dist/*.whl + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - run: uv venv .venv-gateway && uv pip install --python .venv-gateway dist/*.whl + + - name: Smoke test the installed gateway binary + run: | + printf 'model_list: []\n' > "$RUNNER_TEMP/config.yaml" + LITELLM_CONFIG="$RUNNER_TEMP/config.yaml" HOST=127.0.0.1 PORT=4010 \ + .venv-gateway/bin/litellm-gateway > "$RUNNER_TEMP/gateway.log" 2>&1 & + gateway_pid=$! + trap 'kill "$gateway_pid" 2>/dev/null || true; cat "$RUNNER_TEMP/gateway.log"' EXIT + curl --fail --silent --retry 30 --retry-delay 1 --retry-all-errors --retry-connrefused \ + http://127.0.0.1:4010/health/readiness diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index b4b7afb9a09..cff1fa17959 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -42,7 +42,7 @@ jobs: id: rust-bridge-key if: steps.changes.outputs.decision != 'skip' env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }} + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/gateway/pyproject.toml', 'litellm-rust/crates/**', 'pyproject.toml') }} run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - name: Set up Python @@ -73,15 +73,16 @@ jobs: timeout-minutes: 5 uses: ./.github/actions/cache-cargo-build - - name: Build editable Rust bridge + - name: Build editable Rust bridge and gateway wheel if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' - timeout-minutes: 15 + timeout-minutes: 20 run: | mkdir -p rust-bridge-dist uv run --no-project --with maturin==1.15.0 python -c 'import maturin; maturin.build_editable("rust-bridge-dist")' strip --strip-debug litellm/rust_bridge/_native.abi3.so mkdir -p rust-bridge-dist/litellm/rust_bridge cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so + uv build --package litellm-gateway --wheel --out-dir rust-bridge-dist - name: Upload Rust bridge artifact if: steps.changes.outputs.decision != 'skip' @@ -1001,7 +1002,7 @@ jobs: if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-gateway uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1439,7 +1440,7 @@ jobs: if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-gateway uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/litellm-rust/crates/gateway/pyproject.toml b/litellm-rust/crates/gateway/pyproject.toml new file mode 100644 index 00000000000..f93d12889b6 --- /dev/null +++ b/litellm-rust/crates/gateway/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "litellm-gateway" +version = "0.1.0" +description = "Native gateway binary for the LiteLLM Proxy" +requires-python = ">=3.10, <3.16" +license = "MIT" + +[project.urls] +Repository = "https://github.com/BerriAI/litellm" + +[build-system] +requires = ["maturin==1.15.0"] +build-backend = "maturin" + +[tool.maturin] +manifest-path = "Cargo.toml" +bindings = "bin" +profile = "release" +strip = true + +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "../../Cargo.toml" }, + { file = "../../Cargo.lock" }, + { file = "../**/*" }, + { file = "../../../rust-toolchain.toml" }, + { file = "../../../.cargo/config.toml" }, +] + +[tool.commitizen] +version = "0.1.0" +version_files = [ + "pyproject.toml:^version", +] diff --git a/pyproject.toml b/pyproject.toml index 2d25c1ec737..675de2c9806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -385,7 +385,7 @@ litellm-proxy-extras = { workspace = true } litellm-enterprise = { workspace = true } [tool.uv.workspace] -members = ["enterprise", "litellm-proxy-extras"] +members = ["enterprise", "litellm-proxy-extras", "litellm-rust/crates/gateway"] [tool.commitizen] version = "1.109.0" diff --git a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py index 0eae041f535..cdd6109f236 100644 --- a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py @@ -57,26 +57,32 @@ def _write_wheel( dist_info: str = _DIST_INFO, duplicate_wheel: bool = False, native_bytes: bytes = b"synthetic native extension", + kind: str = "extension", ) -> Path: - wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" + distribution: Final = "litellm_gateway" if kind == "gateway" else "litellm" + metadata_directory: Final = f"{distribution}-1.100.0.dist-info" if kind == "gateway" else dist_info + native_member: Final = ( + f"{distribution}-1.100.0.data/scripts/litellm-gateway" if kind == "gateway" else _NATIVE_MEMBER + ) + wheel: Final = tmp_path / f"{distribution}-1.100.0-{filename_tag}.whl" with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: - archive.writestr(_NATIVE_MEMBER, native_bytes) + archive.writestr(native_member, native_bytes) archive.writestr( - f"{dist_info}/METADATA", - "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", + f"{metadata_directory}/METADATA", + f"Metadata-Version: 2.1\nName: {distribution}\nVersion: 1.100.0\n", ) archive.writestr( - f"{dist_info}/RECORD", - f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n", + f"{metadata_directory}/RECORD", + f"{native_member},,\n{metadata_directory}/WHEEL,,\n", ) if metadata_tags is not None: wheel_metadata: Final = ( "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n" + "".join(f"Tag: {tag}\n" for tag in metadata_tags) ) - archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + archive.writestr(f"{metadata_directory}/WHEEL", wheel_metadata) if duplicate_wheel: - archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + archive.writestr(f"{metadata_directory}/WHEEL", wheel_metadata) return wheel @@ -106,6 +112,9 @@ def _run_verifier( wheel: Path, *, exposes_panic: bool = False, + kind: str = "extension", + loads: bool = True, + run_command: _CommandRunner = _fake_subprocess_run, ) -> int: native_module: Final = ( _NativeModuleWithPanicHook("litellm.rust_bridge._native") @@ -113,15 +122,16 @@ def _run_verifier( else ModuleType("litellm.rust_bridge._native") ) - def _fake_load_native_module(_: Path) -> ModuleType: - return native_module + def _fake_load_native_module(_: Path) -> ModuleType | None: + assert kind != "gateway", "A standalone binary must not be imported as a Python extension" + return native_module if loads else None environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) return verifier.main( - (str(_MODULE_PATH), str(wheel)), + (str(_MODULE_PATH), kind, str(wheel)), environment, _fake_load_native_module, - _fake_subprocess_run, + run_command, ) @@ -210,3 +220,77 @@ def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) assert _run_verifier(wheel) == (1 if embedded else 0) + + +@pytest.mark.parametrize("kind, limit", (("extension", 48_000_000), ("gateway", 37_000_000))) +@pytest.mark.parametrize("excess", (0, 1), ids=("at-limit", "over-limit")) +def test_enforces_uncompressed_native_size_limit(tmp_path: Path, kind: str, limit: int, excess: int) -> None: + tag: Final = "py3-none-linux_x86_64" if kind == "gateway" else _EXPECTED_TAG + wheel: Final = _write_wheel( + tmp_path, filename_tag=tag, metadata_tags=(tag,), native_bytes=b"x" * (limit + excess), kind=kind + ) + + assert _run_verifier(wheel, kind=kind) == (0 if excess == 0 else 1) + summary: Final = (tmp_path / "summary.md").read_text() + label: Final = "Gateway binary" if kind == "gateway" else "Native extension" + assert f"| {label} | {(limit + excess) / 1_000_000:.2f} MB |" in summary + + +def test_keeps_both_artifact_reports_in_summary(tmp_path: Path) -> None: + extension: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + gateway: Final = _write_wheel( + tmp_path, filename_tag="py3-none-linux_x86_64", metadata_tags=("py3-none-linux_x86_64",), kind="gateway" + ) + + assert _run_verifier(extension) == 0 + assert _run_verifier(gateway, kind="gateway") == 0 + summary: Final = (tmp_path / "summary.md").read_text() + assert "## Native wheel build report" in summary + assert "## Gateway wheel build report" in summary + + +@pytest.mark.parametrize("kind, other", (("extension", "gateway"), ("gateway", "extension"))) +def test_rejects_wheel_under_wrong_profile(tmp_path: Path, kind: str, other: str) -> None: + tag: Final = "py3-none-linux_x86_64" if kind == "gateway" else _EXPECTED_TAG + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,), kind=kind) + + assert _run_verifier(wheel, kind=other) == 1 + + +def test_rejects_unknown_profile(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel, kind="unknown") == 2 + + +@pytest.mark.parametrize("arguments", ((), ("extension",), ("extension", "wheel.whl", "extra"))) +def test_rejects_wrong_argument_count(arguments: tuple[str, ...]) -> None: + assert verifier.main((str(_MODULE_PATH), *arguments), None, lambda _: None, _fake_subprocess_run) == 2 + + +def test_load_failure_does_not_report_a_panic_hook_failure(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel, loads=False) == 1 + summary: Final = (tmp_path / "summary.md").read_text() + assert summary.count("| X |") == 1 + assert "| Native module loads | Yes | X |" in summary + assert "native module did not load" in capsys.readouterr().err + + +def test_preserves_size_report_when_readelf_fails(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + def _failing_readelf( + command: tuple[str, ...], *, check: bool, capture_output: bool, text: bool + ) -> subprocess.CompletedProcess[str]: + if command[0] == "readelf": + raise subprocess.CalledProcessError(1, command) + return _fake_subprocess_run(command, check=check, capture_output=capture_output, text=text) + + with pytest.raises(subprocess.CalledProcessError): + _run_verifier(wheel, run_command=_failing_readelf) + + summary: Final = (tmp_path / "summary.md").read_text() + assert "## Native wheel build report" in summary + assert "| Compressed wheel |" in summary diff --git a/uv.lock b/uv.lock index b40490112cf..c63b619f116 100644 --- a/uv.lock +++ b/uv.lock @@ -21,6 +21,7 @@ brotli = "2026-10-22T14:01:00Z" members = [ "litellm", "litellm-enterprise", + "litellm-gateway", "litellm-proxy-extras", ] constraints = [ @@ -5478,6 +5479,11 @@ name = "litellm-enterprise" version = "0.1.80" source = { editable = "enterprise" } +[[package]] +name = "litellm-gateway" +version = "0.1.0" +source = { editable = "litellm-rust/crates/gateway" } + [[package]] name = "litellm-proxy-extras" version = "0.4.111"