From 1677c2250e24697a904dd1750815f574af8040f4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 23:52:32 -0800 Subject: [PATCH] ci: stop PR runs from thrashing the Actions cache and stop smoke compiling from source (#51343) * ci: stop PR runs from writing Actions caches and remove the sccache GitHub cache backend PR runs were saving their own cache copies and sccache was making one cache request per compiled file, about 700 to 2000 per Rust build. Together they pushed the repository past GitHub's cache storage and request limits, so restores failed with 429 or found nothing and jobs rebuilt from scratch. The rust-bridge job that gates every unit shard went from 0% cache misses on Oct 21 to about 65% on Nov 11. Caches are now saved only from main and restored everywhere else. The Rust DB cache gets the same save-if guard the other Rust jobs already have. The lint python and ui-api-types jobs restore the uv cache that the unit workflow saves on main, because lint only runs on pull requests and could never save a cache that another pull request can read. The Terraform provider jobs turn off setup-go's built-in cache, which saves from every ref, and use a new cache-go-build action that follows the existing restore-everywhere and save-on-main pattern. sccache is removed from the unit, Rust and Rust DB workflows. The Cargo target directory caches they already restore stay in place. The smoke diagnostics upload no longer fails a job whose checks passed. * ci(smoke): install the prebuilt Rust bridge and cache wheels built from source Every smoke job compiled the Rust bridge inside uv sync, about 4.5 minutes each in 8 jobs per run, and the Python 3.14 and 3.15 jobs also compiled grpcio-tools, numpy and jiter from source on every run because no wheels exist for those versions yet. Smoke only ran on pull requests, so nothing it built could be saved for the next run. The Linux smoke jobs now restore the bridge that the unit workflow already builds once and saves on main. The bridge is abi3, so the same build serves every Python version. When the restore misses, which happens when a pull request changes a Rust build input, the job restores the Cargo build cache and compiles as before. The cache key and restore step move into a restore-rust-bridge action that the unit workflow's producer job also uses, so the key is defined once. Smoke now also runs on pushes to main and prunes the uv cache down to wheels built from source before setup-uv saves it. Pull requests restore that cache, so 3.14 and 3.15 stop recompiling the same packages. The windows-sdk job drops its Cargo cache step. Nothing ever saved a Windows copy, and saving one from main would add about 2 GB to a cache that is already near its limit. --- .github/actions/cache-go-build/action.yml | 39 ++++++++++++ .../actions/restore-rust-bridge/action.yml | 37 +++++++++++ .github/workflows/test-linting.yml | 8 +-- .github/workflows/test-merge-smoke.yml | 62 +++++++++++++++++-- .github/workflows/test-rust-db.yml | 10 +-- .github/workflows/test-rust.yml | 19 ------ .github/workflows/test-terraform-provider.yml | 18 ++++-- .github/workflows/test-unit.yml | 28 +-------- 8 files changed, 154 insertions(+), 67 deletions(-) create mode 100644 .github/actions/cache-go-build/action.yml create mode 100644 .github/actions/restore-rust-bridge/action.yml diff --git a/.github/actions/cache-go-build/action.yml b/.github/actions/cache-go-build/action.yml new file mode 100644 index 00000000000..f29b144c934 --- /dev/null +++ b/.github/actions/cache-go-build/action.yml @@ -0,0 +1,39 @@ +name: "Cache the Go build" +description: >- + Restore the Go module and build caches on every run and save them only from + main, so pull requests reuse main's cache instead of evicting it with their + own copies. + + actions/setup-go's built-in cache saves from every ref, and a cache saved by a + pull request is readable only by that pull request. Callers must pass + `cache: false` to setup-go and run this action after it. + +inputs: + dependency-path: + description: "Path to the go.sum that keys the cache" + required: true + +runs: + using: composite + steps: + - name: Restore and save the Go module and build caches + if: github.ref == 'refs/heads/main' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Restore the Go module and build caches + if: github.ref != 'refs/heads/main' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- diff --git a/.github/actions/restore-rust-bridge/action.yml b/.github/actions/restore-rust-bridge/action.yml new file mode 100644 index 00000000000..d050431dbc6 --- /dev/null +++ b/.github/actions/restore-rust-bridge/action.yml @@ -0,0 +1,37 @@ +name: "Restore the Rust bridge" +description: >- + Restore the prebuilt Rust bridge that the unit workflow's rust-bridge job saves + on main, keyed on every input of the Rust build. + + The bridge is abi3, so one build serves every supported Python version. A job + whose Rust inputs match main installs the restored wheel and native library + instead of compiling them inside `uv sync`. On a miss the caller builds from + source as before. + + The restored `rust-bridge-dist` directory holds the editable litellm wheel, the + litellm-proxy gateway wheel, and `litellm/rust_bridge/_native.abi3.so`. + +outputs: + key: + description: "Cache key for the bridge built from this checkout" + value: ${{ steps.key.outputs.key }} + cache-hit: + description: "'true' when the bridge for this exact key was restored" + value: ${{ steps.restore.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Define the Rust bridge cache key + id: key + shell: bash + env: + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} + run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" + + - name: Restore the Rust bridge + id: restore + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: rust-bridge-dist + key: ${{ steps.key.outputs.key }} diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index f401d022858..2358cbfc028 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -73,14 +73,14 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.decision != 'skip' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv .venv - key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-lint- + ${{ runner.os }}-uv- - name: Clean Python cache if: steps.changes.outputs.decision != 'skip' @@ -396,7 +396,7 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.relevant == 'true' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 362d56c314e..8ed1876a081 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -3,6 +3,8 @@ name: smoke on: pull_request: branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"] + push: + branches: [main] workflow_dispatch: permissions: @@ -52,8 +54,26 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies - run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" --no-install-project + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/litellm-*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Create the loopback-only network namespace run: | @@ -87,6 +107,7 @@ jobs: - name: Upload smoke diagnostics if: always() + continue-on-error: true uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: merge-smoke-diagnostics-py${{ matrix.python-version }} @@ -97,6 +118,10 @@ jobs: if: always() run: sudo ip netns delete smoke + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + all-extras: name: all-extras-py${{ matrix.python-version }} runs-on: ubuntu-24.04 @@ -123,12 +148,34 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install every extra and dependency group - run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" --no-install-project --no-install-package litellm-proxy + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Import the SDK, the proxy server and the enterprise package run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise" + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + windows-sdk: name: windows-sdk runs-on: windows-2025 @@ -154,15 +201,16 @@ jobs: with: version: "0.10.9" - - name: Set up the Rust bridge - uses: ./.github/actions/rust-bridge - - name: Install dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11 - name: Run the Windows tests run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + proxy-build-from-pip: name: proxy-build-from-pip runs-on: ubuntu-24.04 @@ -298,6 +346,10 @@ jobs: if: always() run: cat /tmp/litellm-pip.log || true + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + smoke-passed: name: smoke passed needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip] diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index 6d4eba7b25e..116d6785287 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -42,10 +42,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -91,11 +87,7 @@ jobs: with: workspaces: litellm/rust cache-on-failure: true - - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Python dependencies working-directory: . diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 74a6957aba4..880e8ea785c 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -78,10 +78,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -108,11 +104,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: cargo clippy --workspace --all-targets --locked -- -D warnings rust-test: @@ -155,11 +146,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Check generated trace contracts working-directory: . run: uv run litellm/scripts/generate_trace_types.py --check @@ -206,11 +192,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: uv build --wheel --out-dir dist - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 7dc61f52364..34958526c59 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -4,6 +4,7 @@ on: push: paths: - "litellm/deploy/terraform/provider/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" pull_request: branches: @@ -13,6 +14,7 @@ on: - "litellm/deploy/terraform/provider/**" - "litellm/proxy/**" - "litellm/enterprise/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" permissions: @@ -38,8 +40,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: gofmt run: | @@ -120,8 +126,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: Audit provider endpoints against the schema working-directory: litellm/deploy/terraform/provider diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4c048ace26a..f27b58149ab 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -41,13 +41,6 @@ jobs: timeout-minutes: 2 uses: ./.github/actions/detect-changes - - name: Define editable Rust bridge cache key - id: rust-bridge-key - if: steps.changes.outputs.decision != 'skip' - env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} - run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - - name: Set up Python if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 @@ -66,30 +59,17 @@ jobs: id: rust-bridge-cache if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 - with: - path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + uses: ./.github/actions/restore-rust-bridge - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 5 uses: ./.github/actions/cache-cargo-build - - name: Set up sccache - if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Build editable Rust bridge if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 15 env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: | mkdir -p rust-bridge-dist @@ -103,10 +83,6 @@ jobs: timeout-minutes: 15 working-directory: litellm/packaging/litellm-proxy env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../../rust-bridge-dist @@ -125,7 +101,7 @@ jobs: uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + key: ${{ steps.rust-bridge-cache.outputs.key }} - name: Expose the Rust bridge artifact id: rust-bridge-artifact