From c877876895e33e85ef3a49d119fb9d41aa03553b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 7 Sep 2026 16:39:44 -0700 Subject: [PATCH] refactor(ci): isolate boundary benchmarks --- .github/actions/cache-cargo-build/action.yml | 30 +++++ .../actions/setup-python-boundary/action.yml | 15 ++- .github/actions/setup-rust/action.yml | 28 ---- .github/workflows/_test-unit-base.yml | 4 +- .github/workflows/check-ui-api-types.yml | 4 +- .github/workflows/codspeed.yml | 88 +++++++----- .github/workflows/e2e_record_replay.yml | 8 +- .github/workflows/mutation-test.yml | 4 +- .../publish-basedpyright-base-counts.yml | 4 +- .github/workflows/test-code-quality.yml | 4 +- .github/workflows/test-e2e-changed.yml | 4 +- .github/workflows/test-linting.yml | 4 +- .github/workflows/test-mcp.yml | 4 +- .github/workflows/test-rust.yml | 126 ++++++++---------- .github/workflows/test-terraform-provider.yml | 4 +- .github/workflows/test-unit-documentation.yml | 4 +- .github/workflows/weekly_load_anomaly.yml | 4 +- .../python-bridge/src/routes/benchmarks.rs | 1 - .../python-bridge/src/routes/definition.rs | 123 ++++++++--------- .../crates/python-bridge/src/routes/mod.rs | 1 + tests/benchmarks/python_boundary/README.md | 23 +--- 21 files changed, 246 insertions(+), 241 deletions(-) create mode 100644 .github/actions/cache-cargo-build/action.yml delete mode 100644 .github/actions/setup-rust/action.yml diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml new file mode 100644 index 00000000000..c3b8ce22c68 --- /dev/null +++ b/.github/actions/cache-cargo-build/action.yml @@ -0,0 +1,30 @@ +name: "Cache the Rust build" +description: >- + Cache the Cargo registry and target directory the root package's build needs, + so only the first job on a given Cargo.lock compiles the bridge from scratch. + + litellm builds through maturin, which compiles litellm-rust/crates/python-bridge + in the dev profile for editable installs. `uv sync` therefore pays a full build + in every job that installs the workspace. Nothing caught it, because the uv cache + holds wheels uv downloads rather than wheels it builds, and a path dependency + whose source moves every commit could never hit that cache anyway. Cargo rebuilds + only what changed when its target directory survives, so a warm job pays for the + bridge crate alone. + + The key namespace is separate from test-rust.yml's check and release caches. They + cache the same directory for different workloads, and a shared key would let + whichever ran first deny the others a save. + +runs: + using: composite + steps: + - name: Restore the Cargo registry and target directory + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-maturin-dev- diff --git a/.github/actions/setup-python-boundary/action.yml b/.github/actions/setup-python-boundary/action.yml index da293a07bed..bcbc7628751 100644 --- a/.github/actions/setup-python-boundary/action.yml +++ b/.github/actions/setup-python-boundary/action.yml @@ -9,9 +9,20 @@ runs: - uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - - uses: ./.github/actions/setup-rust + - name: Require preinstalled Rustup + shell: bash + run: | + if ! command -v rustup >/dev/null; then + echo "::error::This runner must provide Rustup" + exit 1 + fi + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: - cache-namespace: boundary-release-bench + cache-shared-key: boundary-release-bench + cache-workspace-crates: "true" + cache-workspaces: litellm-rust -> target + rust-src-dir: ${{ github.workspace }} + rustflags: "" - name: Build and install release benchmark wheel shell: bash run: | diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml deleted file mode 100644 index 5dd2016c26c..00000000000 --- a/.github/actions/setup-rust/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Set up Rust -description: Install the pinned Rust toolchain and cache a Cargo workspace -inputs: - cache-namespace: - description: Stable cache namespace shared by jobs with the same build profile - required: false - default: maturin-dev - cache-workspaces: - description: Cargo workspace and target directory pairs to cache - required: false - default: litellm-rust -> target -runs: - using: composite - steps: - - name: Require preinstalled Rustup - shell: bash - run: | - if ! command -v rustup >/dev/null; then - echo "::error::This runner must provide Rustup; remote-script installation is disabled" - exit 1 - fi - - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - with: - cache-shared-key: ${{ inputs.cache-namespace }}-workspace-v1-${{ runner.arch }} - cache-workspace-crates: "true" - cache-workspaces: ${{ inputs.cache-workspaces }} - rust-src-dir: ${{ github.workspace }} - rustflags: "" diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 0e8cc959925..75b0f93fd77 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -104,10 +104,10 @@ jobs: restore-keys: | ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}- - - name: Set up Rust + - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 - uses: ./.github/actions/setup-rust + uses: ./.github/actions/cache-cargo-build - name: Install dependencies if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml index cbf627ba278..312a80103f8 100644 --- a/.github/workflows/check-ui-api-types.yml +++ b/.github/workflows/check-ui-api-types.yml @@ -67,9 +67,9 @@ jobs: restore-keys: | ${{ runner.os }}-uv- - - name: Set up Rust + - name: Cache the Rust build if: steps.changes.outputs.relevant == 'true' - uses: ./.github/actions/setup-rust + uses: ./.github/actions/cache-cargo-build - name: Install backend dependencies if: steps.changes.outputs.relevant == 'true' diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index c675c9eca5c..fd8a0f4c12b 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -5,27 +5,46 @@ on: branches: - main - litellm_internal_staging + paths: + - "litellm/**" + - "litellm-rust/**" + - "tests/benchmarks/**" + - "pyproject.toml" + - "uv.lock" + - "rust-toolchain.toml" + - ".github/workflows/codspeed.yml" + - ".github/actions/setup-uv-with-retries/**" + - ".github/actions/cache-cargo-build/**" + - ".github/actions/setup-python-boundary/**" pull_request: branches: - main - litellm_internal_staging + paths: + - "litellm/**" + - "litellm-rust/**" + - "tests/benchmarks/**" + - "pyproject.toml" + - "uv.lock" + - "rust-toolchain.toml" + - ".github/workflows/codspeed.yml" + - ".github/actions/setup-uv-with-retries/**" + - ".github/actions/cache-cargo-build/**" + - ".github/actions/setup-python-boundary/**" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: permissions: contents: read + id-token: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - python-suite: - name: Python suite (CPU) - permissions: - contents: read - id-token: write + benchmarks: runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -37,50 +56,53 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.12.12" + python-version: "3.12" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build # Build the wheel and resolve every dependency outside the CodSpeed # runner: the same maturin build took 42 minutes inside `codspeed run` # versus under 3 minutes as a plain step (LIT-6183) - name: Build environment - env: - PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" - run: | - uv sync --frozen --no-default-groups --python python - uv pip install --python .venv/bin/python \ - pytest==8.3.5 pytest-codspeed==4.3.0 mcp==1.28.1 a2a-sdk==1.1.0 - - - name: Collect benchmarks - env: - PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" - run: >- - .venv/bin/python -m pytest -p pytest_codspeed.plugin - tests/benchmarks/ --ignore=tests/benchmarks/python_boundary - --codspeed --collect-only -q + run: > + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 + uv run --frozen --no-default-groups + --with pytest==8.3.5 + --with pytest-codspeed==4.3.0 + --with "mcp>=1.26.0,<2.0" + --with "a2a-sdk>=1.1.0,<2.0" + pytest + -p pytest_codspeed.plugin + tests/benchmarks/ + --ignore=tests/benchmarks/python_boundary + --codspeed + --collect-only -q - name: Run benchmarks uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation - run: >- + run: > env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - .venv/bin/python -m pytest -p pytest_codspeed.plugin - tests/benchmarks/ --ignore=tests/benchmarks/python_boundary + uv run --frozen --no-default-groups + --with pytest==8.3.5 + --with pytest-codspeed==4.3.0 + --with "mcp>=1.26.0,<2.0" + --with "a2a-sdk>=1.1.0,<2.0" + pytest + -p pytest_codspeed.plugin + tests/benchmarks/ + --ignore=tests/benchmarks/python_boundary --codspeed boundary-cpu: name: Python boundary (simulation) - permissions: - contents: read - id-token: write runs-on: ubuntu-24.04 timeout-minutes: 30 env: @@ -98,13 +120,15 @@ jobs: run: >- .venv-boundary/bin/python -m pytest -c /dev/null --rootdir=. -p no:cacheprovider -p pytest_codspeed.plugin --confcutdir=tests/benchmarks/python_boundary - tests/benchmarks/python_boundary/test_boundary.py -k "python_to_rust or rust_to_python or conversion_roundtrip" --codspeed + tests/benchmarks/python_boundary/test_boundary.py::test_python_to_rust + tests/benchmarks/python_boundary/test_boundary.py::test_rust_to_python + tests/benchmarks/python_boundary/test_boundary.py::test_conversion_roundtrip + tests/benchmarks/python_boundary/test_boundary.py::test_rust_to_python_response + --codspeed boundary-latency: name: Python boundary (walltime) - permissions: - contents: read - id-token: write + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false runs-on: codspeed-macro timeout-minutes: 30 env: diff --git a/.github/workflows/e2e_record_replay.yml b/.github/workflows/e2e_record_replay.yml index 79a5581e23a..ca52f0b4d81 100644 --- a/.github/workflows/e2e_record_replay.yml +++ b/.github/workflows/e2e_record_replay.yml @@ -61,8 +61,8 @@ jobs: with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | @@ -159,8 +159,8 @@ jobs: with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index f74e08a0219..b7d28bcaae4 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -53,8 +53,8 @@ jobs: restore-keys: | ${{ runner.os }}-uv- - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml index 0ceec98c9a7..27d4682dbd9 100644 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ b/.github/workflows/publish-basedpyright-base-counts.yml @@ -43,8 +43,8 @@ jobs: with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Cache Prisma binaries uses: ./.github/actions/cache-prisma-binaries diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 4bb187faa6f..e6d2264fbf0 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -56,8 +56,8 @@ jobs: restore-keys: | ${{ runner.os }}-uv- - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: uv sync --frozen --all-groups --all-extras diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index a9acc6243cb..23ab6dfcfe4 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -106,8 +106,8 @@ jobs: with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index c6414a8dd28..9f1283da19e 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -89,9 +89,9 @@ jobs: run: | uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - - name: Set up Rust + - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' - uses: ./.github/actions/setup-rust + uses: ./.github/actions/cache-cargo-build - name: Install dependencies if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index bec2350de6a..93ffcbe0586 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -47,9 +47,9 @@ jobs: with: version: "0.10.9" - - name: Set up Rust + - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' - uses: ./.github/actions/setup-rust + uses: ./.github/actions/cache-cargo-build - name: Install dependencies if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 47152b92f62..08f76169fc6 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -7,10 +7,10 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" + - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - - ".github/actions/setup-rust/**" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -23,10 +23,10 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" + - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - - ".github/actions/setup-rust/**" - ".github/workflows/test-rust.yml" permissions: @@ -36,102 +36,94 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + CARGO_TERM_COLOR: always + jobs: - rust-checks: - name: rustfmt, clippy, test + rust-lint: runs-on: ubuntu-latest timeout-minutes: 10 defaults: run: working-directory: litellm-rust - env: - CARGO_TERM_COLOR: always - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - - name: Set up Rust - uses: ./.github/actions/setup-rust + - run: rustup toolchain install --no-self-update + + - run: cargo fmt --check + + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - cache-namespace: rust-checks + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ github.job }}- - - name: Check Rust formatting - run: cargo fmt --check + - run: cargo clippy --workspace --all-targets --locked -- -D warnings - - name: Run Clippy - run: cargo clippy --workspace --all-targets --locked -- -D warnings + - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings - - name: Run Clippy with Bedrock auth - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings + - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings - - name: Run Clippy with all gateway features - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings + - run: cargo clippy -p litellm-python-bridge --all-targets --features bench,trace-parity --locked -- -D warnings - - name: Check and test bridge feature combinations - run: | - for features in "" bench trace-parity bench,trace-parity; do - cargo clippy -p litellm-python-bridge --all-targets --features "$features" --locked -- -D warnings - cargo test -p litellm-python-bridge --features "$features" --locked - done - - - name: Run Rust tests - run: cargo test --workspace --locked - - - name: Run core tests with Bedrock auth - run: cargo test -p litellm-core --features bedrock-auth --locked - - # Not --all-features: python-config links libpython, which this job does not install. - - name: Run gateway tests with the server feature - run: cargo test -p litellm-ai-gateway --features server --locked - - release-wheel: - name: release wheel + rust-test: runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - env: - CARGO_TERM_COLOR: always + timeout-minutes: 30 steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries + - uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - run: rustup toolchain install --no-self-update + + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - cache-namespace: release-wheel + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ github.job }}- - - name: Build release wheel - run: uv build --wheel --out-dir dist + - run: cargo test --workspace --locked + working-directory: litellm-rust - - name: Build panic contract wheel - run: >- + - run: cargo test -p litellm-core --features bedrock-auth --locked + working-directory: litellm-rust + + - run: cargo test -p litellm-ai-gateway --features server --locked + working-directory: litellm-rust + + - run: cargo test -p litellm-python-bridge --features bench,trace-parity --locked + working-directory: litellm-rust + + - run: uv build --wheel --out-dir dist + + - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl + + - run: >- uv build --wheel --out-dir panic-dist --config-setting "maturin.build-args=--features panic-test,extension-module" - - name: Smoke-test native panic unwinding - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - - - name: Verify stripped native extension - env: - RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - - - name: Test native route wheel - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl + - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 547652ec004..eb7b299fd1f 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -88,8 +88,8 @@ jobs: restore-keys: | ${{ runner.os }}-uv- - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 9c538e65658..90b6b28374e 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -67,9 +67,9 @@ jobs: restore-keys: | ${{ runner.os }}-uv- - - name: Set up Rust + - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' - uses: ./.github/actions/setup-rust + uses: ./.github/actions/cache-cargo-build - name: Install dependencies if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 2b9bd77a5a0..3e1fca89645 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -47,8 +47,8 @@ jobs: with: version: "0.10.9" - - name: Set up Rust - uses: ./.github/actions/setup-rust + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build - name: Install dependencies run: | diff --git a/litellm-rust/crates/python-bridge/src/routes/benchmarks.rs b/litellm-rust/crates/python-bridge/src/routes/benchmarks.rs index eed8eb7de0d..5e2dc809e18 100644 --- a/litellm-rust/crates/python-bridge/src/routes/benchmarks.rs +++ b/litellm-rust/crates/python-bridge/src/routes/benchmarks.rs @@ -23,7 +23,6 @@ bridge_route! { optional = {}, prepare = prepare_echo, errors = core_error_to_pyerr, - trace = disabled, } #[pyclass(frozen)] diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index 5dd993dc2e2..bc51647cbad 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -3,8 +3,6 @@ use pyo3::prelude::*; use pyo3::types::PyCFunction; macro_rules! bridge_route { - (@trace [disabled] $($item:item)*) => {}; - (@trace [] $($item:item)*) => { $($item)* }; ( sync = $sync_name:ident, asynchronous = $async_name:ident, @@ -14,7 +12,6 @@ macro_rules! bridge_route { prepare = $prepare:path, errors = $map_error:path $(, extra = [$($extra:ident),* $(,)?])? - $(, trace = $trace:ident)? $(,)? ) => { struct $inputs { @@ -61,72 +58,70 @@ macro_rules! bridge_route { Ok(()) } - bridge_route! { @trace [$($trace)?] - #[cfg(feature = "trace-parity")] - mod trace { - use pyo3::prelude::*; - use super::{$inputs, $map_error, $prepare}; + #[cfg(feature = "trace-parity")] + mod trace { + use pyo3::prelude::*; + use super::{$inputs, $map_error, $prepare}; - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $sync_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_sync( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $async_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_async( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - pub(super) fn register( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($sync_name, module)?, - )?; - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($async_name, module)?, - )?; - Ok(()) - } + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] + #[allow(clippy::too_many_arguments)] + fn $sync_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type,)* + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::execution::run_sync( + py, + $crate::function_trace::capture(future), + $map_error, + ) } - #[cfg(feature = "trace-parity")] - pub(super) fn register_trace( + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] + #[allow(clippy::too_many_arguments)] + fn $async_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type,)* + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::execution::run_async( + py, + $crate::function_trace::capture(future), + $map_error, + ) + } + + pub(super) fn register( module: &pyo3::Bound<'_, pyo3::types::PyModule>, ) -> pyo3::PyResult<()> { - trace::register(module) + $crate::routes::definition::add_function( + module, + pyo3::wrap_pyfunction!($sync_name, module)?, + )?; + $crate::routes::definition::add_function( + module, + pyo3::wrap_pyfunction!($async_name, module)?, + )?; + Ok(()) } } + + #[cfg(feature = "trace-parity")] + pub(super) fn register_trace( + module: &pyo3::Bound<'_, pyo3::types::PyModule>, + ) -> pyo3::PyResult<()> { + trace::register(module) + } }; } @@ -492,9 +487,7 @@ asyncio.run(exercise()) r#" result = routes.echo("traced") assert result["response"] == "traced", result -assert [(event["id"], event["parent_id"], event["function"]) for event in result["trace"]] == [ - (0, None, "execute_echo"), -], result +assert [event["function"] for event in result["trace"]] == ["execute_echo"], result failure = routes.echo("error") assert failure["error"] == "invalid request: synthetic error", failure assert [event["function"] for event in failure["trace"]] == ["execute_echo"], failure diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index ba125f46abc..a4d0cd87437 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -12,6 +12,7 @@ mod messages; mod ocr; #[cfg(feature = "bench")] +#[allow(dead_code)] mod benchmarks; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/tests/benchmarks/python_boundary/README.md b/tests/benchmarks/python_boundary/README.md index d5ac169b08f..1b5fead6865 100644 --- a/tests/benchmarks/python_boundary/README.md +++ b/tests/benchmarks/python_boundary/README.md @@ -22,20 +22,13 @@ PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv-boundary/bin/python -m pytest -c /dev/nul The existing Criterion serialization benchmark remains available through `cargo bench -p litellm-python-bridge --bench serialization` from `litellm-rust` -## Merge protection - -After the integration branch has a successful baseline, configure the repository's CodSpeed regression threshold and require `CodSpeed Performance Analysis` in branch protection - - ## Rust feature coverage -CI runs bridge Clippy and tests with default features, `bench`, `trace-parity`, and `bench,trace-parity`. The benchmark echo route disables generated trace wrappers; production routes keep their existing trace behavior. `extension-module` is checked through wheel builds because it changes Python linking +CI runs the existing default workspace checks plus bridge Clippy and tests with `bench,trace-parity`. The private benchmark namespace does not register trace wrappers. `extension-module` is checked through wheel builds because it changes Python linking ## Reproducible environments and caches -All three benchmark jobs use Python 3.12.12. The Python suite prepares its environment with `uv sync --frozen` and installs pinned benchmark dependencies before collecting or measuring tests. Measurement invokes the prepared interpreter directly, without dependency resolution - -Update Python pins in both setup locations together. For the Python suite, copy `mcp` and `a2a-sdk` versions from `uv.lock` when updating their pins. Keep pytest and pytest-codspeed aligned with the boundary requirements +The boundary jobs use Python 3.12.12 and install only the hash-locked benchmark dependencies. The existing Python suite keeps its current environment and measurements unchanged To update boundary dependencies, edit `requirements.in`, regenerate the hash-locked file, then run the local validation above: @@ -43,14 +36,4 @@ To update boundary dependencies, edit `requirements.in`, regenerate the hash-loc uv pip compile --python-version 3.12.12 --generate-hashes --no-header tests/benchmarks/python_boundary/requirements.in -o tests/benchmarks/python_boundary/requirements.txt ``` -The shared Rust setup requires preinstalled Rustup and reads `rust-toolchain.toml`. Cargo caches include workspace crates and dependencies, separated by workload, architecture, and toolchain. The `workspace-v1` cache generation avoids restoring older dependency-only entries. Compare two runs of the same commit to verify restoration and compilation savings; a cold cache must also pass - -## Enforcement rollout - -After this PR merges normally, verify that `litellm_internal_staging` has a successful CodSpeed baseline with 25 boundary simulation measurements and 45 boundary walltime measurements. Apply a 10% per-benchmark regression threshold to these new measurements, preserving existing Python-suite thresholds. Ensure regression failures are not configured as informational checks - -Use a temporary, unmerged PR targeting the integration branch to introduce repeated conversion work in an existing boundary benchmark without changing its identifier. Confirm CodSpeed reports a regression, remove the extra work, and confirm recovery. Close the validation PR after collecting both results - -Only after baseline and failure/recovery validation, require `CodSpeed Performance Analysis` on `litellm_internal_staging`, preserving existing protections. Leave `main` unchanged. Verify an unchanged-code rerun passes. The existing `guard-internal-staging` ruleset (ID `17360296`) owns required status checks. Update that rule in place rather than creating a competing protection rule. Repository and CodSpeed administration access are required; an inaccessible classic branch-protection API is not evidence that rulesets are absent - -See [threshold configuration](https://codspeed.io/docs/features/customization) and [branch protection](https://codspeed.io/docs/features/performance-checks) +The boundary setup requires preinstalled Rustup, reads `rust-toolchain.toml`, and caches its release build independently from existing CI caches