Merge 8aec166f0e into cfc651dcd5

.github/scripts/verify_linux_native_wheel.py

@@ -7,14 +7,62 @@ import subprocess
 import sys
 import zipfile
 from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
 from itertools import product
 from pathlib import Path, PurePosixPath
 from types import MappingProxyType, ModuleType
 from typing import Final, Protocol
 
-EXPECTED_PYTHON_TAG: Final = "cp310"
-EXPECTED_ABI_TAG: Final = "abi3"
 EXPECTED_PLATFORM_TAG: Final = "linux_x86_64"
+REPO_ROOT: Final = Path(__file__).parents[2]
+
+
+@dataclass(frozen=True, slots=True)
+class WheelProfile:
+    distribution: str
+    python_tag: str
+    abi_tag: str
+    project_directory: Path
+    label: str
+    report_title: str
+    size_limit: int
+    is_payload: Callable[[str], bool]
+    is_extension: bool
+
+
+PROFILES: Final = MappingProxyType(
+    {
+        "extension": WheelProfile(
+            distribution="litellm",
+            python_tag="cp310",
+            abi_tag="abi3",
+            project_directory=REPO_ROOT,
+            label="Native extension",
+            report_title="Native wheel build report",
+            size_limit=48_000_000,
+            is_payload=lambda name: name.startswith("litellm/rust_bridge/_native.") and name.endswith(".so"),
+            is_extension=True,
+        ),
+        "gateway": WheelProfile(
+            distribution="litellm_gateway",
+            python_tag="py3",
+            abi_tag="none",
+            project_directory=REPO_ROOT / "litellm-rust" / "crates" / "gateway",
+            label="Gateway binary",
+            report_title="Gateway wheel build report",
+            size_limit=37_000_000,
+            is_payload=lambda name: name.endswith(".data/scripts/litellm-gateway"),
+            is_extension=False,
+        ),
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Validation:
+    label: str
+    passed: bool
+    failure: str
 
 
 class CommandRunner(Protocol):
@@ -65,6 +113,57 @@ def _load_native_module(native_path: Path) -> ModuleType | None:
     return native_module
 
 
+def _extension_validations(
+    wheel: Path,
+    native_path: Path,
+    native_bytes: bytes,
+    load_native_module: Callable[[Path], ModuleType | None],
+    run_command: CommandRunner,
+) -> tuple[Validation, ...]:
+    with zipfile.ZipFile(wheel) as archive:
+        duplicated_vocabularies: Final = tuple(
+            member.filename
+            for member in archive.infolist()
+            if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
+            and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
+            and member.file_size > 0
+            and archive.read(member) in native_bytes
+        )
+    dynamic_symbols: Final = run_command(
+        ("readelf", "--dyn-syms", "--wide", str(native_path)),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    native_module: Final = load_native_module(native_path)
+    return (
+        Validation(
+            "Tokenizer vocabularies are not duplicated in the native extension",
+            not duplicated_vocabularies,
+            f"native extension duplicates tokenizer vocabularies: {', '.join(duplicated_vocabularies)}",
+        ),
+        Validation(
+            "Python extension entry point is present",
+            "PyInit__native" in dynamic_symbols,
+            "native extension does not export PyInit__native",
+        ),
+        Validation("Native module loads", native_module is not None, "native module did not load"),
+        Validation(
+            "Production module omits the panic test hook",
+            native_module is None or not hasattr(native_module, "_panic_for_test"),
+            "production native module exposes _panic_for_test",
+        ),
+    )
+
+
+def _write_report(report: str, summary_path: str | None) -> None:
+    if summary_path is None:
+        sys.stdout.write(report + "\n")
+        return
+    with Path(summary_path).open("a") as summary:
+        summary.write(report + "\n")
+
+
 def main(
     argv: Sequence[str] | None = None,
     environment: Mapping[str, str] | None = None,
@@ -73,18 +172,19 @@ def main(
 ) -> int:
     arguments: Final = tuple(sys.argv if argv is None else argv)
     resolved_environment: Final = os.environ if environment is None else environment
-    if len(arguments) != 2:
-        sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n")
+    if len(arguments) != 3 or arguments[1] not in PROFILES:
+        sys.stderr.write("usage: verify_linux_native_wheel.py {extension|gateway} WHEEL\n")
         return 2
 
-    wheel: Final = Path(arguments[1])
+    profile: Final = PROFILES[arguments[1]]
+    wheel: Final = Path(arguments[2])
     wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3)
     if len(wheel_tags) != 4:
         sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n")
         return 1
 
     wheel_identity: Final = wheel_tags[0].split("-")
-    if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]:
+    if len(wheel_identity) != 2 or wheel_identity[0] != profile.distribution or not wheel_identity[1]:
         sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n")
         return 1
 
@@ -115,13 +215,9 @@ def main(
             member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL"
         )
         wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members)
-        native_members: Final = tuple(
-            member
-            for member in wheel_members
-            if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
-        )
+        native_members: Final = tuple(member for member in wheel_members if profile.is_payload(member.filename))
         if len(native_members) != 1:
-            sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
+            sys.stderr.write(f"expected one {profile.label.lower()}, found {len(native_members)}\n")
             return 1
 
         unexpected_members: Final = tuple(
@@ -136,14 +232,6 @@ def main(
         native_path.parent.mkdir(parents=True, exist_ok=True)
         native_bytes: Final = archive.read(native_member)
         native_path.write_bytes(native_bytes)
-        duplicated_vocabularies: Final = tuple(
-            member.filename
-            for member in wheel_members
-            if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
-            and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
-            and member.file_size > 0
-            and archive.read(member) in native_bytes
-        )
 
     wheel_metadata_tags_match: Final = (
         len(wheel_metadata_tags) == len(expanded_filename_tags)
@@ -158,7 +246,7 @@ def main(
         capture_output=True,
         text=True,
     ).stdout.strip()
-    pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text()
+    pyproject: Final = (profile.project_directory / "pyproject.toml").read_text()
     maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject)
     if maturin_match is None:
         sys.stderr.write("build-system does not pin an exact Maturin version\n")
@@ -168,7 +256,7 @@ def main(
     native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100
     size_report: Final = "\n".join(
         (
-            "## Native wheel build report",
+            f"## {profile.report_title}",
             "",
             "| Build | Value |",
             "| --- | --- |",
@@ -183,16 +271,13 @@ def main(
             "| --- | ---: |",
             f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |",
             f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |",
-            f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |",
+            f"| {profile.label} | {native_member.file_size / 1_000_000:.2f} MB |",
             f"| Native share | {native_percentage:.2f}% |",
             "",
         )
     )
     summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY")
-    if summary_path is None:
-        sys.stdout.write(size_report)
-    else:
-        Path(summary_path).write_text(size_report)
+    _write_report(size_report, summary_path)
 
     sections: Final = run_command(
         ("readelf", "--sections", "--wide", str(native_path)),
@@ -201,92 +286,77 @@ def main(
         text=True,
     ).stdout
     debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections)
-    debug_sections_absent: Final = not debug_sections
-    static_symbol_table_absent: Final = ".symtab" not in sections
-
-    dynamic_symbols: Final = run_command(
-        ("readelf", "--dyn-syms", "--wide", str(native_path)),
-        check=True,
-        capture_output=True,
-        text=True,
-    ).stdout
-    extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols
-    native_module: Final = load_native_module(native_path)
-    native_module_loads: Final = native_module is not None
-    panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 48_000_000
-    native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
-        (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
-        (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG),
-        (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG),
-        ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories),
-        (
+        Validation(
+            f"Python tag is {profile.python_tag}",
+            python_tag == profile.python_tag,
+            f"unexpected Python tag: expected {profile.python_tag}, found {python_tag}",
+        ),
+        Validation(
+            f"ABI tag is {profile.abi_tag}",
+            abi_tag == profile.abi_tag,
+            f"unexpected ABI tag: expected {profile.abi_tag}, found {abi_tag}",
+        ),
+        Validation(
+            f"Platform tag is {EXPECTED_PLATFORM_TAG}",
+            platform_tag == EXPECTED_PLATFORM_TAG,
+            f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
+        ),
+        Validation(
+            "Wheel dist-info directory matches the filename",
+            dist_info_directories == expected_dist_info_directories,
+            f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
+            f"found {', '.join(sorted(dist_info_directories))}",
+        ),
+        Validation(
             "Required dist-info files are present exactly once",
             all(count == 1 for count in dist_info_file_counts.values()),
+            f"required dist-info file counts are invalid: {dist_info_file_counts}",
         ),
-        ("Wheel metadata tags match the filename", wheel_metadata_tags_match),
-        ("Debug sections are absent", debug_sections_absent),
-        ("Static symbol table is absent", static_symbol_table_absent),
-        ("Python extension entry point is present", extension_entry_point_present),
-        ("Native module loads", native_module_loads),
-        ("Production module omits the panic test hook", panic_test_hook_absent),
-        (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
-        ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
-        ("Wheel contents are valid", not unexpected_members),
+        Validation(
+            "Wheel metadata tags match the filename",
+            wheel_metadata_tags_match,
+            f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
+            f"found {', '.join(sorted(wheel_metadata_tags))}",
+        ),
+        Validation(
+            "Debug sections are absent",
+            not debug_sections,
+            f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}",
+        ),
+        Validation(
+            "Static symbol table is absent",
+            ".symtab" not in sections,
+            f"{native_member.filename} contains a static symbol table",
+        ),
+        Validation(
+            f"{profile.label} does not exceed {profile.size_limit / 1_000_000:.0f} MB",
+            native_member.file_size <= profile.size_limit,
+            f"{profile.label.lower()} exceeds {profile.size_limit / 1_000_000:.0f} MB: "
+            f"{native_member.file_size / 1_000_000:.2f} MB",
+        ),
+        Validation(
+            "Wheel contents are valid",
+            not unexpected_members,
+            f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}",
+        ),
+    ) + (
+        _extension_validations(wheel, native_path, native_bytes, load_native_module, run_command)
+        if profile.is_extension
+        else ()
     )
 
-    verified_report: Final = size_report + "\n".join(
+    verified_report: Final = "\n".join(
         ("", "| Validation | Expected | Result |", "| --- | --- | :---: |")
-        + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations)
+        + tuple(f"| {row.label} | Yes | {'O' if row.passed else 'X'} |" for row in validations)
         + ("",)
     )
-    if summary_path is not None:
-        Path(summary_path).write_text(verified_report)
+    _write_report(verified_report, summary_path)
 
-    invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values())
-    validation_errors: Final = tuple(
-        message
-        for failed, message in (
-            (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"),
-            (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"),
-            (not extension_entry_point_present, "native extension does not export PyInit__native"),
-            (
-                python_tag != EXPECTED_PYTHON_TAG,
-                f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}",
-            ),
-            (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"),
-            (
-                platform_tag != EXPECTED_PLATFORM_TAG,
-                f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
-            ),
-            (
-                dist_info_directories != expected_dist_info_directories,
-                f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
-                f"found {', '.join(sorted(dist_info_directories))}",
-            ),
-            (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"),
-            (
-                not invalid_dist_info_files and not wheel_metadata_tags_match,
-                f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
-                f"found {', '.join(sorted(wheel_metadata_tags))}",
-            ),
-            (
-                native_module is not None and not panic_test_hook_absent,
-                "production native module exposes _panic_for_test",
-            ),
-            (
-                not native_size_within_limit,
-                f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
-                f"{native_member.file_size / 1_000_000:.2f} MB",
-            ),
-            (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
-        )
-        if failed
-    )
+    validation_errors: Final = tuple(row.failure for row in validations if not row.passed)
     sys.stderr.write("".join(f"{message}\n" for message in validation_errors))
 
-    return 0 if all(passed for _, passed in validations) else 1
+    return 0 if not validation_errors else 1
 
 
 if __name__ == "__main__":

.github/workflows/test-linting.yml

@@ -171,7 +171,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy
+          uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy --no-install-package litellm-gateway
           uv run --no-sync python scripts/openapi_docs_gate.py --base "$GATE_BASE_SHA"
 
       - name: Print OpenAI version
@@ -412,7 +412,7 @@ jobs:
 
       - name: Install backend dependencies
         if: steps.changes.outputs.relevant == 'true'
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --no-install-package litellm-gateway
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.relevant == 'true'
@@ -516,7 +516,7 @@ jobs:
         uses: ./.github/actions/cache-cargo-build
 
       - name: Install dependencies
-        run: uv sync --frozen --all-groups --all-extras
+        run: uv sync --frozen --all-groups --all-extras --no-install-package litellm-gateway
 
       - name: check_licenses
         run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py

.github/workflows/test-merge-smoke.yml

@@ -52,6 +52,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up Rust before concurrent package builds
+        run: rustup toolchain install --no-self-update
+
       - name: Install dependencies
         run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
 
@@ -100,7 +103,7 @@ jobs:
   all-extras:
     name: all-extras-py${{ matrix.python-version }}
     runs-on: ubuntu-24.04
-    timeout-minutes: 10
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
@@ -123,6 +126,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up Rust before concurrent package builds
+        run: rustup toolchain install --no-self-update
+
       - name: Install every extra and dependency group
         run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
 

.github/workflows/test-rust.yml

@@ -189,7 +189,7 @@ jobs:
 
       - run: uv build --wheel --out-dir dist
 
-      - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
+      - run: python .github/scripts/verify_linux_native_wheel.py extension dist/*.whl
         env:
           RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
@@ -203,3 +203,45 @@ jobs:
           --config-setting "maturin.build-args=--features panic-test,extension-module"
 
       - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl
+
+  gateway-wheel:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - run: rustup toolchain install --no-self-update
+
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          workspaces: litellm-rust
+          cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - run: uv build --package litellm-gateway --wheel --out-dir dist
+
+      - run: python .github/scripts/verify_linux_native_wheel.py gateway dist/*.whl
+        env:
+          RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - run: uv venv .venv-gateway && uv pip install --python .venv-gateway dist/*.whl
+
+      - name: Smoke test the installed gateway binary
+        run: |
+          printf 'model_list: []\n' > "$RUNNER_TEMP/config.yaml"
+          LITELLM_CONFIG="$RUNNER_TEMP/config.yaml" HOST=127.0.0.1 PORT=4010 \
+            .venv-gateway/bin/litellm-gateway > "$RUNNER_TEMP/gateway.log" 2>&1 &
+          gateway_pid=$!
+          trap 'kill "$gateway_pid" 2>/dev/null || true; cat "$RUNNER_TEMP/gateway.log"' EXIT
+          curl --fail --silent --retry 30 --retry-delay 1 --retry-all-errors --retry-connrefused \
+            http://127.0.0.1:4010/health/readiness

.github/workflows/test-unit.yml

@@ -42,7 +42,7 @@ jobs:
         id: rust-bridge-key
         if: steps.changes.outputs.decision != 'skip'
         env:
-          RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
+          RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/gateway/pyproject.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
         run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
 
       - name: Set up Python
@@ -73,15 +73,16 @@ jobs:
         timeout-minutes: 5
         uses: ./.github/actions/cache-cargo-build
 
-      - name: Build editable Rust bridge
+      - name: Build editable Rust bridge and gateway wheel
         if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
-        timeout-minutes: 15
+        timeout-minutes: 20
         run: |
           mkdir -p rust-bridge-dist
           uv run --no-project --with maturin==1.15.0 python -c 'import maturin; maturin.build_editable("rust-bridge-dist")'
           strip --strip-debug litellm/rust_bridge/_native.abi3.so
           mkdir -p rust-bridge-dist/litellm/rust_bridge
           cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
+          uv build --package litellm-gateway --wheel --out-dir rust-bridge-dist
 
       - name: Upload Rust bridge artifact
         if: steps.changes.outputs.decision != 'skip'
@@ -1001,7 +1002,7 @@ jobs:
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
             .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-gateway
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1439,7 +1440,7 @@ jobs:
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
             .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-gateway
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
           fi

litellm-rust/crates/gateway/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "litellm-gateway"
+version = "0.1.0"
+description = "Native gateway binary for the LiteLLM Proxy"
+requires-python = ">=3.10, <3.16"
+license = "MIT"
+
+[project.urls]
+Repository = "https://github.com/BerriAI/litellm"
+
+[build-system]
+requires = ["maturin==1.15.0"]
+build-backend = "maturin"
+
+[tool.maturin]
+manifest-path = "Cargo.toml"
+bindings = "bin"
+profile = "release"
+strip = true
+
+[tool.uv]
+cache-keys = [
+    { file = "pyproject.toml" },
+    { file = "Cargo.toml" },
+    { file = "../../Cargo.toml" },
+    { file = "../../Cargo.lock" },
+    { file = "../**/*" },
+    { file = "../../../rust-toolchain.toml" },
+    { file = "../../../.cargo/config.toml" },
+]
+
+[tool.commitizen]
+version = "0.1.0"
+version_files = [
+    "pyproject.toml:^version",
+]

pyproject.toml

@@ -385,7 +385,7 @@ litellm-proxy-extras = { workspace = true }
 litellm-enterprise = { workspace = true }
 
 [tool.uv.workspace]
-members = ["enterprise", "litellm-proxy-extras"]
+members = ["enterprise", "litellm-proxy-extras", "litellm-rust/crates/gateway"]
 
 [tool.commitizen]
 version = "1.109.0"

tests/unit/rust_bridge/test_verify_linux_native_wheel.py

@@ -57,26 +57,32 @@ def _write_wheel(
     dist_info: str = _DIST_INFO,
     duplicate_wheel: bool = False,
     native_bytes: bytes = b"synthetic native extension",
+    kind: str = "extension",
 ) -> Path:
-    wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl"
+    distribution: Final = "litellm_gateway" if kind == "gateway" else "litellm"
+    metadata_directory: Final = f"{distribution}-1.100.0.dist-info" if kind == "gateway" else dist_info
+    native_member: Final = (
+        f"{distribution}-1.100.0.data/scripts/litellm-gateway" if kind == "gateway" else _NATIVE_MEMBER
+    )
+    wheel: Final = tmp_path / f"{distribution}-1.100.0-{filename_tag}.whl"
     with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive:
-        archive.writestr(_NATIVE_MEMBER, native_bytes)
+        archive.writestr(native_member, native_bytes)
         archive.writestr(
-            f"{dist_info}/METADATA",
-            "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n",
+            f"{metadata_directory}/METADATA",
+            f"Metadata-Version: 2.1\nName: {distribution}\nVersion: 1.100.0\n",
         )
         archive.writestr(
-            f"{dist_info}/RECORD",
-            f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n",
+            f"{metadata_directory}/RECORD",
+            f"{native_member},,\n{metadata_directory}/WHEEL,,\n",
         )
         if metadata_tags is not None:
             wheel_metadata: Final = (
                 "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n"
                 + "".join(f"Tag: {tag}\n" for tag in metadata_tags)
             )
-            archive.writestr(f"{dist_info}/WHEEL", wheel_metadata)
+            archive.writestr(f"{metadata_directory}/WHEEL", wheel_metadata)
             if duplicate_wheel:
-                archive.writestr(f"{dist_info}/WHEEL", wheel_metadata)
+                archive.writestr(f"{metadata_directory}/WHEEL", wheel_metadata)
     return wheel
 
 
@@ -106,6 +112,9 @@ def _run_verifier(
     wheel: Path,
     *,
     exposes_panic: bool = False,
+    kind: str = "extension",
+    loads: bool = True,
+    run_command: _CommandRunner = _fake_subprocess_run,
 ) -> int:
     native_module: Final = (
         _NativeModuleWithPanicHook("litellm.rust_bridge._native")
@@ -113,15 +122,16 @@ def _run_verifier(
         else ModuleType("litellm.rust_bridge._native")
     )
 
-    def _fake_load_native_module(_: Path) -> ModuleType:
-        return native_module
+    def _fake_load_native_module(_: Path) -> ModuleType | None:
+        assert kind != "gateway", "A standalone binary must not be imported as a Python extension"
+        return native_module if loads else None
 
     environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")})
     return verifier.main(
-        (str(_MODULE_PATH), str(wheel)),
+        (str(_MODULE_PATH), kind, str(wheel)),
         environment,
         _fake_load_native_module,
-        _fake_subprocess_run,
+        run_command,
     )
 
 
@@ -210,3 +220,77 @@ def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None:
         archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks)
 
     assert _run_verifier(wheel) == (1 if embedded else 0)
+
+
+@pytest.mark.parametrize("kind, limit", (("extension", 48_000_000), ("gateway", 37_000_000)))
+@pytest.mark.parametrize("excess", (0, 1), ids=("at-limit", "over-limit"))
+def test_enforces_uncompressed_native_size_limit(tmp_path: Path, kind: str, limit: int, excess: int) -> None:
+    tag: Final = "py3-none-linux_x86_64" if kind == "gateway" else _EXPECTED_TAG
+    wheel: Final = _write_wheel(
+        tmp_path, filename_tag=tag, metadata_tags=(tag,), native_bytes=b"x" * (limit + excess), kind=kind
+    )
+
+    assert _run_verifier(wheel, kind=kind) == (0 if excess == 0 else 1)
+    summary: Final = (tmp_path / "summary.md").read_text()
+    label: Final = "Gateway binary" if kind == "gateway" else "Native extension"
+    assert f"| {label} | {(limit + excess) / 1_000_000:.2f} MB |" in summary
+
+
+def test_keeps_both_artifact_reports_in_summary(tmp_path: Path) -> None:
+    extension: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
+    gateway: Final = _write_wheel(
+        tmp_path, filename_tag="py3-none-linux_x86_64", metadata_tags=("py3-none-linux_x86_64",), kind="gateway"
+    )
+
+    assert _run_verifier(extension) == 0
+    assert _run_verifier(gateway, kind="gateway") == 0
+    summary: Final = (tmp_path / "summary.md").read_text()
+    assert "## Native wheel build report" in summary
+    assert "## Gateway wheel build report" in summary
+
+
+@pytest.mark.parametrize("kind, other", (("extension", "gateway"), ("gateway", "extension")))
+def test_rejects_wheel_under_wrong_profile(tmp_path: Path, kind: str, other: str) -> None:
+    tag: Final = "py3-none-linux_x86_64" if kind == "gateway" else _EXPECTED_TAG
+    wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,), kind=kind)
+
+    assert _run_verifier(wheel, kind=other) == 1
+
+
+def test_rejects_unknown_profile(tmp_path: Path) -> None:
+    wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
+
+    assert _run_verifier(wheel, kind="unknown") == 2
+
+
+@pytest.mark.parametrize("arguments", ((), ("extension",), ("extension", "wheel.whl", "extra")))
+def test_rejects_wrong_argument_count(arguments: tuple[str, ...]) -> None:
+    assert verifier.main((str(_MODULE_PATH), *arguments), None, lambda _: None, _fake_subprocess_run) == 2
+
+
+def test_load_failure_does_not_report_a_panic_hook_failure(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
+
+    assert _run_verifier(wheel, loads=False) == 1
+    summary: Final = (tmp_path / "summary.md").read_text()
+    assert summary.count("| X |") == 1
+    assert "| Native module loads | Yes | X |" in summary
+    assert "native module did not load" in capsys.readouterr().err
+
+
+def test_preserves_size_report_when_readelf_fails(tmp_path: Path) -> None:
+    wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
+
+    def _failing_readelf(
+        command: tuple[str, ...], *, check: bool, capture_output: bool, text: bool
+    ) -> subprocess.CompletedProcess[str]:
+        if command[0] == "readelf":
+            raise subprocess.CalledProcessError(1, command)
+        return _fake_subprocess_run(command, check=check, capture_output=capture_output, text=text)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        _run_verifier(wheel, run_command=_failing_readelf)
+
+    summary: Final = (tmp_path / "summary.md").read_text()
+    assert "## Native wheel build report" in summary
+    assert "| Compressed wheel |" in summary

uv.lock

@@ -21,6 +21,7 @@ brotli = "2026-10-22T14:01:00Z"
 members = [
     "litellm",
     "litellm-enterprise",
+    "litellm-gateway",
     "litellm-proxy-extras",
 ]
 constraints = [
@@ -5478,6 +5479,11 @@ name = "litellm-enterprise"
 version = "0.1.80"
 source = { editable = "enterprise" }
 
+[[package]]
+name = "litellm-gateway"
+version = "0.1.0"
+source = { editable = "litellm-rust/crates/gateway" }
+
 [[package]]
 name = "litellm-proxy-extras"
 version = "0.4.111"