ci: harden cargo fetches during maturin builds (#31348)

.cargo/config.toml

@@ -1,3 +1,12 @@
+[http]
+# CI has seen transient crates.io failures from libcurl's HTTP/2 multiplexing
+# during `maturin` metadata resolution. Disable multiplexing and retry more
+# aggressively so editable `uv sync` builds are not failed by one flaky frame.
+multiplexing = false
+
+[net]
+retry = 5
+
 # PyO3 cdylib (`litellm-python-bridge`) links against the host interpreter's
 # symbols, which are not present at link time when building an extension module.
 # On macOS, tell the linker to resolve undefined `_Py*` symbols dynamically at

.circleci/config.yml

@@ -190,6 +190,8 @@ jobs:
     working_directory: ~/project
     environment:
       UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
     steps:
       - checkout
       - run:
@@ -243,7 +245,17 @@ jobs:
             if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
-            uv sync --frozen --group dev --python 3.11
+            for ($attempt = 1; $attempt -le 5; $attempt++) {
+              Write-Host "uv sync attempt $attempt/5"
+              uv sync --frozen --group dev --python 3.11
+              if ($LASTEXITCODE -eq 0) {
+                break
+              }
+              if ($attempt -eq 5) {
+                exit $LASTEXITCODE
+              }
+              Start-Sleep -Seconds 15
+            }
       - run:
           name: Run Windows-specific test
           command: |
@@ -255,6 +267,7 @@ jobs:
           command: |
             $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             cargo --version
+            Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
             uv build --wheel --out-dir dist
             uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
 

.github/scripts/uv_sync_with_retries.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+max_attempts="${UV_SYNC_MAX_ATTEMPTS:-5}"
+delay_seconds="${UV_SYNC_RETRY_DELAY_SECONDS:-15}"
+
+export CARGO_HTTP_MULTIPLEXING="${CARGO_HTTP_MULTIPLEXING:-false}"
+export CARGO_NET_RETRY="${CARGO_NET_RETRY:-5}"
+
+if [[ "$#" -eq 0 ]]; then
+  echo "usage: $0 <uv sync args...>" >&2
+  exit 2
+fi
+
+for attempt in $(seq 1 "${max_attempts}"); do
+  echo "uv sync attempt ${attempt}/${max_attempts}"
+  status=0
+  if uv sync "$@"; then
+    exit 0
+  else
+    status=$?
+  fi
+
+  if [[ "${attempt}" -eq "${max_attempts}" ]]; then
+    echo "uv sync failed after ${max_attempts} attempts" >&2
+    exit "${status}"
+  fi
+
+  echo "uv sync failed; retrying in ${delay_seconds}s..."
+  sleep "${delay_seconds}"
+done

.github/workflows/_test-unit-base.yml

@@ -73,7 +73,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:

.github/workflows/check-ui-api-types.yml

@@ -46,7 +46,7 @@ jobs:
             ${{ runner.os }}-uv-
 
       - name: Install backend dependencies
-        run: uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:

.github/workflows/mutation-test.yml

@@ -55,7 +55,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:

.github/workflows/test-mcp.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install dependencies
         run: |
           uv lock --check
-          uv sync --frozen --group proxy-dev --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router
 
       - name: Run MCP tests
         run: |

.github/workflows/test-unit-documentation.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:

.github/workflows/test-unit-proxy-legacy.yml

@@ -71,7 +71,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:

pyproject.toml

@@ -234,7 +234,7 @@ healthcheck = [
 ]
 
 [build-system]
-requires = ["maturin>=1.9.4,<2"]
+requires = ["maturin==1.9.4"]
 build-backend = "maturin"
 
 [tool.maturin]