Merge upstream/main into the LLM Shield guardrail branch

Upstream #44003 moved tests/test_litellm/proxy/guardrails into
tests/unit/proxy/guardrails, so test_llm_shield_proxy.py moves with it.

.cargo/config.toml

@@ -17,3 +17,6 @@ rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
 
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[env]
+SQLX_OFFLINE = "true"

.circleci/config.yml

@@ -12,6 +12,9 @@ parameters:
   migration_source_sha:
     type: string
     default: ""
+  routing_parity_base:
+    type: string
+    default: ""
 orbs:
   codecov: codecov/codecov@4.0.1
   node: circleci/node@5.1.0 # Add this line to declare the node orb
@@ -138,7 +141,7 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
     steps:
       - run:
           name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
@@ -164,9 +167,29 @@ commands:
             /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
             rustc --version
             cargo --version
+            { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
+  save_cargo_target:
+    steps:
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/debug
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -176,6 +199,9 @@ commands:
       image:
         type: string
         default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+      server_args:
+        type: string
+        default: ""
     steps:
       - run:
           name: Start PostgreSQL
@@ -186,7 +212,7 @@ commands:
               -e POSTGRES_PASSWORD=postgres \
               -e POSTGRES_DB=<< parameters.db_name >> \
               -p 5432:5432 \
-              << parameters.image >>
+              << parameters.image >> << parameters.server_args >>
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
@@ -275,50 +301,11 @@ commands:
             # `uv sync --package litellm-enterprise` here — that overwrites the
             # shared .venv and strips out dev/test deps (pytest, prisma, etc.).
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
-  setup_litellm_test_deps:
+  install_windows_toolchain:
     steps:
-      - checkout
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
       - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
-
-jobs:
-  # Add Windows testing job
-  using_litellm_on_windows:
-    executor:
-      name: win/default
-      shell: powershell.exe
-    working_directory: ~/project
-    environment:
-      UV_PYTHON: "3.11"
-      CARGO_HTTP_MULTIPLEXING: "false"
-      CARGO_NET_RETRY: "5"
-    steps:
-      - checkout
-      - run:
-          name: Install Python
-          command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
-      - run:
-          name: Install Dependencies
-          environment:
-            UV_HTTP_TIMEOUT: "300"
+          name: Install Rust and uv
+          no_output_timeout: 30m
           command: |
             $rustupInit = Join-Path $env:TEMP "rustup-init.exe"
             $rustupVersion = "1.28.2"
@@ -358,6 +345,55 @@ jobs:
             if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
+  setup_litellm_test_deps:
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
+
+jobs:
+  # Add Windows testing job
+  using_litellm_on_windows:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Install Dependencies
+          no_output_timeout: 30m
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             for ($attempt = 1; $attempt -le 5; $attempt++) {
               Write-Host "uv sync attempt $attempt/5"
               uv sync --frozen --group dev --python 3.11
@@ -372,17 +408,69 @@ jobs:
       - run:
           name: Run Windows-specific test
           command: |
-            uv run --no-sync python -m pytest tests/windows_tests/ -v
+            uv run --no-sync python -m pytest --tb=short tests/windows_tests/ -v
+
+  windows_release_wheel:
+    executor:
+      name: win/default
+      shell: powershell.exe
+      size: xlarge
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
       - run:
-          name: Guard against MAX_PATH-busting packaged wheel paths
+          name: Skip job when no windows-release-relevant files changed
+          shell: bash.exe
+          command: bash .circleci/scripts/path_filter.sh windows-release
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Record the Rust build environment for the release cargo cache key
+          command: |
+            & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: |
+            $fingerprints = "litellm-rust/target/release/.fingerprint"
+            if (Test-Path $fingerprints) {
+              Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
+            }
+      - run:
+          name: Build the release wheel and install it under a worst-case MAX_PATH prefix
+          no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
             $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
-            cargo --version
-            Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
             uv build --wheel --out-dir dist
-            uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
+            if ($LASTEXITCODE -ne 0) {
+              exit $LASTEXITCODE
+            }
+            python tests/windows_tests/check_windows_wheel_install.py
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/release
 
   base_sdk_install:
     docker:
@@ -398,6 +486,7 @@ jobs:
       - install_rust
       - run:
           name: Build the wheel
+          no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
@@ -410,6 +499,10 @@ jobs:
             uv venv /tmp/base-sdk --python 3.12
             VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
             /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+      - run:
+          name: Guard against MAX_PATH-busting packaged wheel paths
+          command: |
+            python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
 
   local_testing_part1:
     docker:
@@ -438,6 +531,7 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -457,7 +551,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
@@ -531,7 +625,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
@@ -603,7 +697,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -658,7 +752,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -721,7 +815,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 -k 'router' \
                 -n 4 \
@@ -765,7 +859,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -810,7 +904,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -854,7 +948,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -892,7 +986,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -937,7 +1031,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -981,7 +1075,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1027,7 +1121,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1082,7 +1176,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1116,7 +1210,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1160,7 +1254,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1204,7 +1298,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1248,7 +1342,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1293,7 +1387,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1338,7 +1432,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1372,7 +1466,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 -n 4 \
@@ -1417,7 +1511,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
@@ -1437,61 +1531,6 @@ jobs:
           paths:
             - audio_coverage.xml
             - audio_coverage
-  redis_caching_unit_tests:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - restore_cache:
-          keys:
-            - v1-uv-cache-{{ checksum "uv.lock" }}
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v1-uv-cache-{{ checksum "uv.lock" }}
-      # Run pytest and generate JUnit XML report
-      - run:
-          name: Run tests
-          command: |
-            mkdir -p test-results
-            TEST_FILES=$(printf "%s\n" \
-              tests/local_testing/test_dual_cache.py \
-              tests/local_testing/test_redis_batch_optimizations.py \
-              tests/local_testing/test_redis_increment_with_floor.py \
-              tests/local_testing/test_router_utils.py)
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -s \
-                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-                --junitxml=test-results/junit.xml \
-                --durations=5 -n 2 \
-                --reruns 2 --reruns-delay 1"
-          no_output_timeout: 20m
-      - run:
-          name: Rename the coverage files
-          command: |
-            mv coverage.xml redis_caching_coverage.xml
-            mv .coverage redis_caching_coverage
-
-      # Store test results
-      - store_test_results:
-          path: test-results
-      - persist_to_workspace:
-          root: .
-          paths:
-            - redis_caching_coverage.xml
-            - redis_caching_coverage
   installing_litellm_on_python:
     docker:
       - *python312_image
@@ -1511,7 +1550,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
+            uv run --no-sync python -m pytest --tb=short -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_3_13:
     docker:
@@ -1535,7 +1574,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
+            uv run --no-sync python -m pytest --tb=short -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_v2_migration_resolver:
     docker:
@@ -1566,7 +1605,7 @@ jobs:
       - run:
           name: Run both migration resolvers against Postgres
           command: |
-            uv run --no-sync python -m pytest -vv \
+            uv run --no-sync python -m pytest --tb=short -vv \
               tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \
               tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
 
@@ -1735,7 +1774,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
@@ -1832,7 +1871,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -s -v \
                 --junitxml=test-results/junit.xml \
                 -n 4 \
@@ -1919,7 +1958,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -s -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2002,7 +2041,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2054,7 +2093,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
@@ -2135,7 +2174,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2240,7 +2279,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2312,7 +2351,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2397,7 +2436,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
@@ -2494,7 +2533,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2565,7 +2604,7 @@ jobs:
             TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
                 -vv -s \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
@@ -2595,7 +2634,7 @@ jobs:
       - run:
           name: Combine Coverage
           command: |
-            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
+            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
             uv tool run --from 'coverage[toml]==7.10.6' coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -3095,7 +3134,7 @@ jobs:
           name: Test provider capture and replay harness
           command: |
             mkdir -p test-results/provider-replay-harness
-            uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
+            uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
               --junitxml=test-results/provider-replay-harness/junit.xml \
               tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
               tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
@@ -3108,10 +3147,18 @@ jobs:
     parameters:
       suite:
         type: string
+      mode:
+        type: enum
+        enum: [standard, replica]
+        default: standard
+      parallelism:
+        type: integer
+        default: 1
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: << parameters.parallelism >>
     steps:
       - setup_litellm_test_deps
       - when:
@@ -3142,18 +3189,19 @@ jobs:
                 command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
       - start_redis
       - run:
           name: Run owned integration contracts
-          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >>
           no_output_timeout: 15m
       - run:
           name: Stop owned database and Redis
           when: always
           command: |
-            mkdir -p test-results/integration-<< parameters.suite >>
-            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
-            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
+            mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >>
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true
             docker rm -f postgres-db redis-cache
             test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
       - store_test_results:
@@ -3161,11 +3209,82 @@ jobs:
       - store_artifacts:
           path: test-results
 
+  routing_parity:
+    parameters:
+      suite:
+        type: string
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Check out base product code
+          environment:
+            ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
+          command: |
+            [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
+            git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
+            git rm -r -f --quiet litellm enterprise litellm-proxy-extras
+            git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
+            git reset --quiet
+            test -f litellm/rust_bridge/_native.abi3.so
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
+      - start_redis
+      - run:
+          name: Run base side
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base
+          no_output_timeout: 15m
+      - run:
+          name: Stop base database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/services-<< parameters.suite >>-parity-base
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - run:
+          name: Check out head product code
+          command: |
+            git rm -r -f --quiet litellm enterprise litellm-proxy-extras
+            git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras
+            git reset --quiet
+            test -f litellm/rust_bridge/_native.abi3.so
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
+      - start_redis
+      - run:
+          name: Run head side
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head
+          no_output_timeout: 15m
+      - run:
+          name: Stop head database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/services-<< parameters.suite >>-parity-head
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - run:
+          name: Compare routing parity
+          command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
   unit:
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: 4
     steps:
       - setup_litellm_test_deps
       - run:
@@ -3175,10 +3294,11 @@ jobs:
           name: Run unit tests
           command: |
             mkdir -p test-results/unit
-            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
-            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
+            if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
+            mapfile -t files < <(printf '%s\n' "${shard}")
             set +e
-            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
@@ -3222,119 +3342,101 @@ workflows:
           cron: "17 0,6,12,18 * * *"
           filters:
             branches:
-              only: litellm_internal_staging
+              only: main
     jobs: *migration_jobs
+  routing_parity:
+    when:
+      not:
+        equal: ["", << pipeline.parameters.routing_parity_base >>]
+    jobs:
+      - routing_parity:
+          name: routing-parity-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [management, accounting, database, providers, extensions, cost, mcp]
   integration:
-    unless: << pipeline.parameters.run_migration_tests >>
+    unless:
+      or:
+        - << pipeline.parameters.run_migration_tests >>
+        - not:
+            equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
       - integration_contracts:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, extensions, sdk, cost, browser]
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
+              suite: [management, accounting, database, providers, mcp, sdk, cost, security, browser]
+      - integration_contracts:
+          name: integration-extensions
+          suite: extensions
+          parallelism: 4
+      - integration_contracts:
+          name: integration-<< matrix.suite >>-replica
+          matrix:
+            parameters:
+              suite: [management, database]
+              mode: [replica]
   build_and_test:
-    unless: << pipeline.parameters.run_migration_tests >>
+    unless:
+      or:
+        - << pipeline.parameters.run_migration_tests >>
+        - not:
+            equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
-      - using_litellm_on_windows:
-          filters: &main_branches
-            branches:
-              only:
-                - main
-                - /litellm_.*/
-      - unit:
-          filters: *main_branches
+      - using_litellm_on_windows
+      - windows_release_wheel
+      - unit
       - provider_replay_harness
-      - base_sdk_install:
-          filters: *main_branches
-      - local_testing_part1:
-          filters: *main_branches
-      - local_testing_part2:
-          filters: *main_branches
-      - langfuse_logging_unit_tests:
-          filters: *main_branches
-      - litellm_assistants_api_testing:
-          filters: *main_branches
-      - litellm_router_testing:
-          filters: *main_branches
-      - litellm_router_unit_testing:
-          filters: *main_branches
-      - auth_ui_unit_tests:
-          filters: *main_branches
-      - build_docker_database_image:
-          filters: *main_branches
-      - e2e_ui_testing:
-          filters: *main_branches
-      - e2e_ui_testing_server_root_path:
-          filters: *main_branches
+      - base_sdk_install
+      - local_testing_part1
+      - local_testing_part2
+      - langfuse_logging_unit_tests
+      - litellm_assistants_api_testing
+      - litellm_router_testing
+      - litellm_router_unit_testing
+      - auth_ui_unit_tests
+      - build_docker_database_image
+      - e2e_ui_testing
+      - e2e_ui_testing_server_root_path
       - build_and_test:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - e2e_openai_endpoints:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_logging_guardrails_model_info_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_spend_accuracy_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_multi_instance_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_store_model_in_db_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - proxy_build_from_pip_tests:
-          filters: *main_branches
+      - proxy_build_from_pip_tests
       - proxy_pass_through_endpoint_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_e2e_anthropic_messages_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - llm_translation_testing:
-          filters: *main_branches
-      - realtime_translation_testing:
-          filters: *main_branches
-      - agent_testing:
-          filters: *main_branches
-      - guardrails_testing:
-          filters: *main_branches
-      - google_generate_content_endpoint_testing:
-          filters: *main_branches
-      - llm_responses_api_testing:
-          filters: *main_branches
-      - ocr_testing:
-          filters: *main_branches
-      - search_testing:
-          filters: *main_branches
-      - batches_testing:
-          filters: *main_branches
-      - litellm_utils_testing:
-          filters: *main_branches
-      - pass_through_unit_testing:
-          filters: *main_branches
-      - image_gen_testing:
-          filters: *main_branches
-      - logging_testing:
-          filters: *main_branches
-      - audio_testing:
-          filters: *main_branches
-      - redis_caching_unit_tests:
-          filters: *main_branches
+      - llm_translation_testing
+      - realtime_translation_testing
+      - agent_testing
+      - guardrails_testing
+      - google_generate_content_endpoint_testing
+      - llm_responses_api_testing
+      - ocr_testing
+      - search_testing
+      - batches_testing
+      - litellm_utils_testing
+      - pass_through_unit_testing
+      - image_gen_testing
+      - logging_testing
+      - audio_testing
       - upload-coverage:
           requires:
             - realtime_translation_testing
@@ -3349,7 +3451,6 @@ workflows:
             - image_gen_testing
             - logging_testing
             - audio_testing
-            - redis_caching_unit_tests
             - langfuse_logging_unit_tests
             - local_testing_part1
             - local_testing_part2
@@ -3359,18 +3460,12 @@ workflows:
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - installing_litellm_on_python:
-          filters: *main_branches
-      - installing_litellm_on_python_3_13:
-          filters: *main_branches
-      - installing_litellm_on_python_v2_migration_resolver:
-          filters: *main_branches
+      - installing_litellm_on_python
+      - installing_litellm_on_python_3_13
+      - installing_litellm_on_python_v2_migration_resolver
       - helm_chart_testing:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - test_bad_database_url:
           requires:
             - build_docker_database_image
-          filters: *main_branches

.circleci/scripts/classify_changes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
 
 has_client=false
 has_backend=false
@@ -9,19 +9,24 @@ has_ci=false
 has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
+has_windows_release=false
 outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
     *.md | *.mdx) : ;;
-    pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/test_litellm/test_circleci_path_filter.py | tests/test_litellm/test_detect_changes.py)
+    pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
       has_mcp_dependencies=true ;;
   esac
   case "$file" in
     tests/e2e/*/*.py) : ;;
-    tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
+    tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
       has_provider_harness=true ;;
   esac
+  case "$file" in
+    litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
+      has_windows_release=true ;;
+  esac
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
@@ -31,7 +36,7 @@ while IFS= read -r file || [ -n "$file" ]; do
   case "$file" in
     model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
       has_cost_map=true ;;
-    tests/test_litellm/* | tests/proxy_unit_tests/*) : ;;
+    tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;;
     *) outside_cost_map_set=true ;;
   esac
 done
@@ -46,6 +51,9 @@ case "$category" in
   provider-harness)
     [ "$has_provider_harness" = true ] && echo run || echo skip
     ;;
+  windows-release)
+    [ "$has_windows_release" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/path_filter.sh

@@ -11,7 +11,7 @@ run_full() {
 
 [ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
 
-candidate_bases="main litellm_internal_staging litellm_oss_staging"
+candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}"
 merge_base=""
 for base in $candidate_bases; do
   git fetch --quiet origin "$base" 2>/dev/null || continue

.circleci/scripts/prepare_replica_roles.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+from typing import Final
+from urllib.parse import urlsplit, urlunsplit
+
+import psycopg
+
+DATABASE_URL: Final = os.environ["DATABASE_URL"]
+
+
+def postgres_url() -> str:
+    parsed: Final = urlsplit(DATABASE_URL)
+    return urlunsplit(parsed._replace(path="/postgres"))
+
+
+def main() -> None:
+    with psycopg.connect(postgres_url(), autocommit=True) as admin:
+        admin.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
+        admin.execute("CREATE ROLE litellm_writer LOGIN PASSWORD 'litellm-writer' NOSUPERUSER")
+        admin.execute("CREATE ROLE litellm_reader LOGIN PASSWORD 'litellm-reader' NOSUPERUSER NOINHERIT")
+        admin.execute("ALTER ROLE litellm_reader SET default_transaction_read_only = on")
+        admin.execute("ALTER DATABASE circle_test OWNER TO litellm_writer")
+        admin.execute("GRANT CONNECT ON DATABASE circle_test TO litellm_reader")
+    with psycopg.connect(DATABASE_URL, autocommit=True) as admin:
+        admin.execute("GRANT USAGE ON SCHEMA public TO litellm_reader")
+        admin.execute(
+            "ALTER DEFAULT PRIVILEGES FOR ROLE litellm_writer IN SCHEMA public GRANT SELECT ON TABLES TO litellm_reader"
+        )
+        admin.execute("GRANT SELECT ON ALL TABLES IN SCHEMA public TO litellm_reader")
+
+    parsed: Final = urlsplit(DATABASE_URL)
+    reader_url: Final = urlunsplit(
+        parsed._replace(netloc=f"litellm_reader:litellm-reader@{parsed.hostname}:{parsed.port}")
+    )
+    writer_url: Final = urlunsplit(
+        parsed._replace(netloc=f"litellm_writer:litellm-writer@{parsed.hostname}:{parsed.port}")
+    )
+    with psycopg.connect(reader_url, autocommit=True) as reader:
+        assert reader.execute("SHOW transaction_read_only").fetchone() == ("on",)
+        try:
+            reader.execute("CREATE TABLE integration_readonly_probe (id int)")
+        except psycopg.errors.ReadOnlySqlTransaction:
+            pass
+        else:
+            raise AssertionError("litellm_reader executed a write statement")
+    with psycopg.connect(writer_url, autocommit=True) as writer:
+        assert writer.execute("SELECT current_user").fetchone() == ("litellm_writer",)
+
+
+if __name__ == "__main__":
+    main()

.circleci/scripts/run_integration.sh

@@ -7,9 +7,16 @@ if [ "${GITHUB_ACTIONS:-}" = true ]; then
 fi
 
 suite="${1:?integration suite required}"
-results="test-results/integration-${suite}"
+mode="${2:-standard}"
+side="${3:-}"
+if [ "$mode" = replica ]; then
+  results="test-results/integration-${suite}-replica"
+elif [ "$mode" = parity ]; then
+  results="test-results/parity-${suite}/${side:?parity side required}"
+else
+  results="test-results/integration-${suite}"
+fi
 mkdir -p "$results"
-shard_timeout=11m
 integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')"
 upstream_pid=""
 proxy_pid=""
@@ -19,6 +26,7 @@ guard_created=false
 guard_installed=false
 guard6_created=false
 guard6_installed=false
+egress_cgroup=litellm-integration
 cleanup() {
   original_status=$?
   trap - EXIT INT TERM
@@ -40,14 +48,14 @@ cleanup() {
     fi
   done
   if [ "$guard_installed" = true ]; then
-    sudo iptables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo iptables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard_created" = true ]; then
     sudo iptables -F integration_only || original_status=1
     sudo iptables -X integration_only || original_status=1
   fi
   if [ "$guard6_installed" = true ]; then
-    sudo ip6tables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo ip6tables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard6_created" = true ]; then
     sudo ip6tables -F integration_only || original_status=1
@@ -81,6 +89,20 @@ export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
 
 uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
 
+export INTEGRATION_PROXY_DATABASE_URL=""
+export INTEGRATION_PROXY_READ_REPLICA_URL=""
+export INTEGRATION_ROUTING=""
+if [ "$mode" = replica ] || [ "$mode" = parity ]; then
+  .venv/bin/python .circleci/scripts/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1
+  export INTEGRATION_PROXY_DATABASE_URL="postgresql://litellm_writer:litellm-writer@127.0.0.1:5432/circle_test"
+  export INTEGRATION_PROXY_READ_REPLICA_URL="postgresql://litellm_reader:litellm-reader@127.0.0.1:5432/circle_test"
+fi
+if [ "$mode" = parity ]; then
+  export INTEGRATION_ROUTING=capture
+fi
+
+sudo mkdir -p "/sys/fs/cgroup/$egress_cgroup"
+echo "$$" | sudo tee "/sys/fs/cgroup/$egress_cgroup/cgroup.procs" > /dev/null
 sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
@@ -90,13 +112,13 @@ for service in postgres-db redis-cache; do
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
 sudo iptables -A integration_only -j REJECT
-sudo iptables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo iptables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard_installed=true
 sudo ip6tables -N integration_only
 guard6_created=true
 sudo ip6tables -A integration_only -o lo -j ACCEPT
 sudo ip6tables -A integration_only -j REJECT
-sudo ip6tables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo ip6tables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard6_installed=true
 
 if curl --noproxy '*' --connect-timeout 2 -s http://198.51.100.1 >/dev/null 2>&1; then
@@ -112,6 +134,15 @@ upstream_pid=$!
 if [ "$suite" = cost ]; then
   export INTEGRATION_WORKERS=8
 fi
+if [ "$suite" = mcp ]; then
+  export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1
+fi
+coverage_data="$PWD/$results/coverage/data"
+proxy_command=(.venv/bin/python -m integration._support.proxy)
+if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
+  mkdir -p "$(dirname "$coverage_data")"
+  proxy_command=(.venv/bin/python -m coverage run --rcfile=tests/integration/mcp_coverage.toml -m integration._support.proxy)
+fi
 start_proxy() {
   local port="$1"
   local log_name="$2"
@@ -129,12 +160,18 @@ start_proxy() {
   else
     cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")
   fi
+  local -a database_env=("DATABASE_URL=${INTEGRATION_PROXY_DATABASE_URL:-$DATABASE_URL}")
+  if [ -n "$INTEGRATION_PROXY_READ_REPLICA_URL" ]; then
+    database_env+=("DATABASE_URL_READ_REPLICA=$INTEGRATION_PROXY_READ_REPLICA_URL")
+  fi
   setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
-    DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+    "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+    INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
+    LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
-    AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
-    .venv/bin/python -m integration._support.proxy --config tests/integration/proxy_config.yaml \
+    AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
+    "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
     --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
     --use_prisma_db_push --enforce_prisma_migration_check \
     > "$results/$log_name" 2>&1 &
@@ -146,7 +183,7 @@ proxy_pid="$launched_pid"
 curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \
   -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
   -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json"
-if [ "$suite" = management ]; then
+if [ "$suite" = management ] || [ "$suite" = mcp ]; then
   export INTEGRATION_PEER_URL=http://127.0.0.1:4001
   start_proxy 4001 peer.log
   peer_pid="$launched_pid"
@@ -154,7 +191,7 @@ if [ "$suite" = management ]; then
 fi
 
 if [ "$suite" = providers ]; then
-  INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --noconftest -o addopts= \
+  INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --tb=short --noconftest -o addopts= \
     --strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
     tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
     tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
@@ -176,14 +213,47 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
-timeout --signal=TERM --kill-after=20s "$shard_timeout" env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
+node_files=()
+if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
+  split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
+    | circleci tests split --split-by=timings --timings-type=filename)"
+  read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
+  test "${#node_files[@]}" -gt 0
+  printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
+fi
+
+env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
   DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
   INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
   INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
+  LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
-  .venv/bin/python tests/integration/run.py "$suite" --results "$results"
+  INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
+  INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
+  INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
+
+if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
+  for covered_pid in "$proxy_pid" "$peer_pid"; do
+    [ -n "$covered_pid" ] || continue
+    kill -TERM -- "-$covered_pid"
+    for _ in {1..300}; do
+      kill -0 "$covered_pid" 2>/dev/null || break
+      sleep 0.1
+    done
+    wait "$covered_pid" 2>/dev/null || true
+  done
+  proxy_pid=""
+  peer_pid=""
+  COVERAGE_FILE="$coverage_data" .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
+  COVERAGE_FILE="$coverage_data" .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml \
+    > "$results/coverage/coverage.txt"
+  COVERAGE_FILE="$coverage_data" .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml \
+    -d "$results/coverage/html"
+  tail -n 1 "$results/coverage/coverage.txt"
+fi

.circleci/scripts/unit_selection.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+flag="${1:?usage: unit_selection.sh <codecov flag>}"
+
+legacy_flags=(
+  caching-local
+  core-utils
+  enterprise-package
+  enterprise-routing
+  integrations
+  llm-other-providers
+  llm-vertex-ai
+  mcp-integration
+  misc
+  proxy-db-auth-checks
+  proxy-db-budgets
+  proxy-db-custom-logging
+  proxy-db-db-and-spend
+  proxy-db-endpoints-and-responses
+  proxy-db-guardrails-hooks
+  proxy-db-jwt-and-keys
+  proxy-db-key-generation
+  proxy-db-logging-misc
+  proxy-db-proxy-runtime
+  proxy-db-proxy-server-core
+  proxy-db-proxy-utils
+  proxy-extras
+  proxy-infra
+  responses-caching-types
+)
+
+legacy_paths() {
+  case "$1" in
+    caching-local) echo tests/unit/caching ;;
+    core-utils) echo tests/unit/litellm_core_utils ;;
+    enterprise-package)
+      echo tests/unit/enterprise/integrations
+      echo tests/unit/enterprise/proxy/auth
+      echo tests/unit/enterprise/proxy/guardrails
+      echo tests/unit/enterprise/proxy/hooks
+      echo tests/unit/enterprise/proxy/management_endpoints
+      echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
+      echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
+    enterprise-routing)
+      echo tests/unit/google_genai
+      echo tests/unit/router_strategy
+      echo tests/unit/router_utils
+      echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
+      echo tests/unit/enterprise/enterprise_callbacks/send_emails
+      echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+      echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
+      echo tests/unit/enterprise/proxy/test_enterprise_routes.py
+      echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
+      echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
+      echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
+    integrations) echo tests/unit/integrations ;;
+    llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
+    llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
+    mcp-integration)
+      echo tests/unit/experimental_mcp_client
+      echo tests/unit/proxy/_experimental/mcp_server
+      echo tests/unit/responses/mcp
+      echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
+    misc)
+      find tests/unit -maxdepth 1 -name 'test_*.py'
+      echo tests/unit/test_router
+      echo tests/unit/a2a_protocol
+      echo tests/unit/batches
+      echo tests/unit/chat_completions
+      echo tests/unit/completion_extras
+      echo tests/unit/containers
+      echo tests/unit/embeddings
+      echo tests/unit/endpoints
+      echo tests/unit/files
+      echo tests/unit/harness
+      echo tests/unit/images
+      echo tests/unit/interactions
+      echo tests/unit/messages
+      echo tests/unit/rag
+      echo tests/unit/rerank_api
+      echo tests/unit/rust_bridge
+      echo tests/unit/secret_managers
+      echo tests/unit/vector_stores
+      echo tests/unit/videos ;;
+    proxy-db-auth-checks)
+      echo tests/unit/proxy/auth/test_auth_checks.py
+      echo tests/unit/proxy/auth/test_user_api_key_auth.py
+      echo tests/unit/proxy/test_credential_slot_registry.py
+      echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
+    proxy-db-budgets)
+      echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+      echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
+      echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
+    proxy-db-custom-logging)
+      echo tests/unit/proxy/test_custom_callback_input.py
+      echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
+    proxy-db-db-and-spend)
+      echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
+      echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
+      echo tests/unit/proxy/db/test_update_daily_tag_spend.py
+      echo tests/unit/proxy/test_db_schema_changes.py
+      echo tests/unit/proxy/test_prisma_client_backoff_retry.py
+      echo tests/unit/proxy/test_update_spend.py
+      echo tests/unit/skills/test_skills_db.py ;;
+    proxy-db-endpoints-and-responses)
+      echo tests/unit/proxy/lens
+      echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
+      echo tests/unit/proxy/common_utils/test_check_batch_cost.py
+      echo tests/unit/proxy/common_utils/test_check_responses_cost.py
+      echo tests/unit/proxy/common_utils/test_realtime_cache.py
+      echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
+      echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
+      echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
+      echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
+      echo tests/unit/proxy/response_polling
+      echo tests/unit/proxy/test_custom_tokenizer_bug.py
+      echo tests/unit/proxy/test_get_favicon.py
+      echo tests/unit/proxy/test_get_image.py
+      echo tests/unit/proxy/test_prompt_test_endpoint.py
+      echo tests/unit/proxy/test_reducto_ocr_route.py
+      echo tests/unit/proxy/test_response_polling_pre_call_checks.py
+      echo tests/unit/proxy/test_ui_path_detection.py ;;
+    proxy-db-guardrails-hooks)
+      echo tests/unit/proxy/hooks/test_banned_keyword_list.py
+      echo tests/unit/proxy/test_proxy_setting_guardrails.py
+      echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
+    proxy-db-jwt-and-keys)
+      echo tests/unit/proxy/auth/test_jwt.py
+      echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
+      echo tests/unit/proxy/test_proxy_custom_auth.py ;;
+    proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
+    proxy-db-logging-misc)
+      echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
+      echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
+      echo tests/unit/proxy/test_proxy_reject_logging.py ;;
+    proxy-db-proxy-runtime)
+      echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
+      echo tests/unit/proxy/auth/test_proxy_routes.py
+      echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
+      echo tests/unit/proxy/test_proxy_config_unit_test.py
+      echo tests/unit/proxy/test_proxy_token_counter.py
+      echo tests/unit/proxy/test_server_root_path.py ;;
+    proxy-db-proxy-server-core)
+      echo tests/unit/proxy/test__lazy_features.py
+      echo tests/unit/proxy/test_aproxy_startup.py
+      echo tests/unit/proxy/test_proxy_server.py ;;
+    proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
+    proxy-extras) echo tests/unit/litellm_proxy_extras ;;
+    proxy-infra)
+      echo tests/unit/gateway
+      echo tests/unit/proxy/management
+      echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+      echo tests/unit/proxy/roi_calculator ;;
+    responses-caching-types)
+      find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
+      echo tests/unit/types ;;
+    *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
+  esac
+}
+
+expand() {
+  while read -r path; do
+    if [ -d "$path" ]; then
+      find "$path" -name 'test_*.py'
+    elif [ -f "$path" ]; then
+      echo "$path"
+    else
+      echo "unit_selection.sh: $path does not exist" >&2
+      exit 1
+    fi
+  done
+}
+
+if [ "$flag" = unit ]; then
+  comm -23 \
+    <(find tests/unit -name 'test_*.py' | sort) \
+    <(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
+  exit 0
+fi
+
+legacy_paths "$flag" | expand | sort

.circleci/scripts/verify_integration_browser.py

@@ -31,8 +31,8 @@ def main() -> None:
     result: Final = json.loads(Path(sys.argv[1]).read_text())
     assert not result.get("errors"), result.get("errors")
     expected: Final = json.loads(
-        (Path(__file__).resolve().parents[2] / "tests/integration/contracts.json").read_text()
-    )["browser"]
+        (Path(__file__).resolve().parents[2] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text()
+    )
     assert expected and result["stats"]["expected"] == len(expected)
     assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped"))
 

.circleci/tests.yml

@@ -0,0 +1,416 @@
+version: 2.1
+
+commands:
+  wait_for_service:
+    parameters:
+      url:
+        type: string
+      timeout:
+        type: string
+        default: "60"
+    steps:
+      - run:
+          name: "Wait for << parameters.url >>"
+          command: |
+            TIMEOUT=<< parameters.timeout >>
+            URL="<< parameters.url >>"
+            ELAPSED=0
+            echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
+            if echo "$URL" | grep -q '^tcp://'; then
+              HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
+              PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
+              while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
+                sleep 2; ELAPSED=$((ELAPSED+2))
+                if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
+              done
+            else
+              while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
+                sleep 2; ELAPSED=$((ELAPSED+2))
+                if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
+              done
+            fi
+            echo "Service ready after ${ELAPSED}s"
+  install_uv:
+    steps:
+      - run:
+          name: Install uv (pinned 0.10.9)
+          command: |
+            curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
+            echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f  /tmp/uv-install.sh" | sha256sum -c -
+            env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
+            rm -f /tmp/uv-install.sh
+            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="$HOME/.local/bin:$PATH"
+  install_rust:
+    steps:
+      - run:
+          name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
+          command: |
+            case "$(uname -m)" in
+              x86_64)
+                RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
+                RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
+                ;;
+              aarch64)
+                RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
+                RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
+                ;;
+              *)
+                echo "install_rust: unsupported architecture $(uname -m)" >&2
+                exit 1
+                ;;
+            esac
+            curl -sSLf -o /tmp/rustup-init \
+              "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
+            echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
+            chmod +x /tmp/rustup-init
+            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
+            rm -f /tmp/rustup-init
+            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="$HOME/.cargo/bin:$PATH"
+            rustc --version
+            cargo --version
+  install_codecov_cli:
+    steps:
+      - run:
+          name: Install Codecov CLI (pinned v11.3.1)
+          when: always
+          command: |
+            curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
+            curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
+            [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d  codecov" ]
+            (cd /tmp && sha256sum -c codecov.SHA256SUM)
+            chmod +x /tmp/codecov
+            mkdir -p "$HOME/.local/bin"
+            mv /tmp/codecov "$HOME/.local/bin/codecov"
+  setup_litellm_enterprise_pip:
+    steps:
+      - run:
+          name: "Install local version of litellm-enterprise"
+          command: |
+            uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
+  setup_test_deps:
+    steps:
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Generate Prisma client
+          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+  skip_unless_relevant:
+    parameters:
+      category:
+        type: string
+        default: backend
+      base_ref:
+        type: string
+        default: ""
+      pull_request_url:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: "Skip job when no << parameters.category >>-relevant files changed"
+          command: |
+            export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}"
+            export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>"
+            [ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH
+            bash .circleci/scripts/path_filter.sh << parameters.category >>
+  start_postgres:
+    parameters:
+      db_name:
+        type: string
+        default: circle_test
+      image:
+        type: string
+        default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+    steps:
+      - run:
+          name: Start PostgreSQL
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=<< parameters.db_name >> \
+              -p 5432:5432 \
+              << parameters.image >>
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
+  start_redis:
+    steps:
+      - run:
+          name: Start Redis
+          command: |
+            docker run -d \
+              --name redis-cache \
+              -p 6379:6379 \
+              redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
+      - wait_for_service:
+          url: tcp://localhost:6379
+          timeout: "60"
+
+jobs:
+  unit:
+    parameters:
+      flag:
+        type: string
+        default: unit
+      shards:
+        type: integer
+        default: 6
+      workers:
+        type: integer
+        default: 4
+      dist:
+        type: string
+        default: loadscope
+      base_ref:
+        type: string
+        default: ""
+      pull_request_url:
+        type: string
+        default: ""
+      legacy_mcp_peer:
+        type: boolean
+        default: false
+      reruns:
+        type: integer
+        default: 0
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    parallelism: << parameters.shards >>
+    environment:
+      COVERAGE_CORE: sysmon
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+    steps:
+      - checkout
+      - skip_unless_relevant:
+          base_ref: << parameters.base_ref >>
+          pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
+      - when:
+          condition: << parameters.legacy_mcp_peer >>
+          steps:
+            - run:
+                name: Install MCP SDK1 peer
+                command: |
+                  uv venv --python 3.12 .venv-mcp-peer
+                  uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+                  echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
+      - run:
+          name: "Run << parameters.flag >> shard"
+          no_output_timeout: 20m
+          command: |
+            mkdir -p test-results/<< parameters.flag >>
+            selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
+            [ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
+            shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
+            [ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
+            mapfile -t files < <(printf '%s\n' "${shard}")
+            xdist_args=()
+            if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
+            rerun_args=(-p no:rerunfailures)
+            if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
+            test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
+            if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
+            set +e
+            env -i "${test_env[@]}" \
+              uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
+            status=$?
+            set -e
+            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
+            exit "$status"
+      - install_codecov_cli
+      - run:
+          name: Upload coverage
+          when: always
+          command: |
+            [ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; }
+            codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+      - store_artifacts:
+          path: coverage.xml
+  documentation:
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_test_deps
+      - run:
+          name: Checkout litellm-docs
+          command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website
+      - run:
+          name: Run documentation validation
+          command: |
+            uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+            uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
+            uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
+            uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
+  integration:
+    parameters:
+      suite:
+        type: string
+      base_ref:
+        type: string
+        default: ""
+      pull_request_url:
+        type: string
+        default: ""
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - checkout
+      - skip_unless_relevant:
+          base_ref: << parameters.base_ref >>
+          pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+      - start_redis
+      - run:
+          name: Run owned integration contracts
+          command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          no_output_timeout: 15m
+      - run:
+          name: Stop owned database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/integration-<< parameters.suite >>
+            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
+workflows:
+  tests:
+    when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_"))
+    jobs:
+      - unit:
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-<< matrix.flag >>
+          shards: 1
+          workers: 2
+          reruns: 2
+          matrix:
+            parameters:
+              flag: [caching-local, proxy-extras, enterprise-routing]
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-mcp-integration
+          flag: mcp-integration
+          shards: 1
+          workers: 2
+          legacy_mcp_peer: true
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-<< matrix.flag >>
+          shards: 1
+          reruns: 2
+          matrix:
+            parameters:
+              flag:
+                - enterprise-package
+                - proxy-infra
+                - responses-caching-types
+                - proxy-db-auth-checks
+                - proxy-db-jwt-and-keys
+                - proxy-db-proxy-server-core
+                - proxy-db-proxy-runtime
+                - proxy-db-custom-logging
+                - proxy-db-logging-misc
+                - proxy-db-db-and-spend
+                - proxy-db-guardrails-hooks
+                - proxy-db-budgets
+                - proxy-db-endpoints-and-responses
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-llm-vertex-ai
+          flag: llm-vertex-ai
+          shards: 2
+          workers: 1
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-llm-other-providers
+          flag: llm-other-providers
+          shards: 3
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-core-utils
+          flag: core-utils
+          shards: 2
+          reruns: 1
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-integrations
+          flag: integrations
+          shards: 2
+          reruns: 3
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-misc
+          flag: misc
+          shards: 2
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-proxy-db-proxy-utils
+          flag: proxy-db-proxy-utils
+          shards: 1
+          reruns: 2
+          dist: worksteal
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-proxy-db-key-generation
+          flag: proxy-db-key-generation
+          shards: 1
+          workers: 0
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - documentation
+      - integration:
+          name: integration-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [sdk]
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>

.githooks/commit-msg

@@ -42,7 +42,7 @@ case "$subject" in
         ;;
 esac
 
-ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert"
+ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert|security"
 # Description must not start with an uppercase letter — kept in sync with the
 # subjectPattern in .github/workflows/conventional-commits.yml so the local
 # hook is the strictly tighter of the two gates. (Without this guard, a commit
@@ -61,7 +61,7 @@ cat >&2 <<EOF
   Expected: <type>(<scope>)!: <description>
             (description must start with a lowercase letter)
 
-  Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert
+  Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert, security
   Examples:
     feat(router): add weighted round-robin strategy
     fix(bedrock): decouple STS region from aws_region_name

.githooks/pre-push

@@ -8,7 +8,6 @@
 #
 # Protected branches (always allowed):
 #   - main
-#   - litellm_internal_staging
 #   - dependabot/*
 #   - gh-readonly-queue/*
 #
@@ -22,7 +21,7 @@ ZERO_OID_SHA256="000000000000000000000000000000000000000000000000000000000000000
 ALLOWED_TYPES="feature|bugfix|hotfix|release|chore"
 BRANCH_PATTERN="^(${ALLOWED_TYPES})/.+"
 
-PROTECTED_NAMES="main litellm_internal_staging"
+PROTECTED_NAMES="main"
 PROTECTED_PREFIXES="dependabot/ gh-readonly-queue/"
 
 is_protected() {
@@ -78,8 +77,7 @@ if [ -n "$invalid" ]; then
     chore/bump-deps
     hotfix/auth-bypass
 
-  Protected (always allowed): main, litellm_internal_staging,
-  dependabot/*, gh-readonly-queue/*.
+  Protected (always allowed): main, dependabot/*, gh-readonly-queue/*.
 
 See https://conventional-branch.github.io/
 

.github/CODEOWNERS

@@ -1,10 +1,2 @@
-/ui/ @yuneng-berri @ryan-crabbe-berri
-/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
-/ui/Dockerfile
-/ui/nginx.conf
-/ui/litellm-dashboard/src/lib/http/schema.d.ts
-/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
-/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
-/.github/CODEOWNERS @yuneng-berri

.github/actions/cache-cargo-build/action.yml

@@ -25,6 +25,7 @@ runs:
   using: composite
   steps:
     - name: Restore the Cargo registry and target directory
+      if: github.ref == 'refs/heads/main'
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         path: |
@@ -34,3 +35,15 @@ runs:
         key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
         restore-keys: |
           ${{ runner.os }}-maturin-${{ inputs.profile }}-
+
+    - name: Restore the Cargo registry and target directory
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          litellm-rust/target
+        key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-maturin-${{ inputs.profile }}-

.github/actions/cache-prisma-binaries/action.yml

@@ -30,6 +30,7 @@ runs:
         echo "version=${version}" >> "$GITHUB_OUTPUT"
 
     - name: Restore Prisma binaries
+      if: github.ref == 'refs/heads/main'
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         # ~/.cache/prisma-python holds the npm install tree prisma-client-py
@@ -38,3 +39,12 @@ runs:
           ~/.cache/prisma-python
           ~/.cache/prisma
         key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}
+
+    - name: Restore Prisma binaries
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cache/prisma-python
+          ~/.cache/prisma
+        key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}

.github/actions/cache-uv-downloads/action.yml

@@ -0,0 +1,25 @@
+name: "Cache uv downloads"
+description: >-
+  Restore the uv download cache on every run and save it only from main, so pull
+  requests reuse main's cache instead of evicting it with their own copies.
+
+runs:
+  using: composite
+  steps:
+    - name: Restore and save the uv download cache
+      if: github.ref == 'refs/heads/main'
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: ${{ env.UV_CACHE_DIR }}
+        key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
+
+    - name: Restore the uv download cache
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: ${{ env.UV_CACHE_DIR }}
+        key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-

.github/actions/setup-uv-with-retries/action.yml

@@ -17,6 +17,7 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}
 
     - name: Wait before attempt 2
       if: steps.attempt-1.outcome == 'failure'
@@ -30,6 +31,7 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}
 
     - name: Wait before attempt 3
       if: steps.attempt-2.outcome == 'failure'
@@ -41,3 +43,4 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}

.github/ci-coverage-allowlist.yml

@@ -4,18 +4,27 @@ description: >-
   by a job nor listed here, so every entry below is a decision on the record.
 
 test_paths:
+  - reason: >-
+      litellm.agent() end-to-end suite. It drives the real claude, codex and opencode CLIs and
+      deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus
+      LITELLM_PROXY_API_BASE / LITELLM_PROXY_API_KEY, and skips without them. Run manually
+      before changing litellm/harness; the mocked coverage runs in tests/unit/harness and
+      tests/unit/llms/*/harness
+    paths:
+      - tests/harness_e2e
   - reason: >-
       The Rust/Python parity harness is run manually through its local CLI. Recorded replay,
       fixture generation, and harness checks are intentionally outside pull request CI
     paths:
       - tests/rust-python-harness
   - reason: >-
-      What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
-      globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
-      not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
-      and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
-      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
-      run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
+      Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
+      glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
+      not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
+      Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
+      test_caching.py by node ID. It does not run that file's other tests.
+      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
+      run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
       credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
       test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
       test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
@@ -110,3 +119,10 @@ dockerfiles:
       An example image under cookbook/ that is documentation rather than a shipped artifact
     paths:
       - cookbook/litellm-ollama-docker-image/Dockerfile
+  - reason: >-
+      The Rust gateway image compiles the whole workspace in release mode, which is too slow for
+      a per-pull-request job while the gateway binary is still being assembled; the Rust lint,
+      clippy, and compile jobs already cover the code it packages. Revisit when the gateway is
+      published
+    paths:
+      - litellm-rust/crates/gateway/Dockerfile

.github/e2e-stack/select_tests.py

@@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
     r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
     r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
     r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
+    r"|^tests/e2e/logging/test_s3_log_e2e\.py$"
     r"|^tests/e2e/secret_manager/"
 )
 HARNESS: Final = re.compile(

.github/merge-smoke-tests.json

@@ -0,0 +1,15 @@
+{
+  "cases": {
+    "CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
+    "CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
+    "CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
+    "MODEL-ALLOW": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_allows_listed_model_for_key",
+    "MODEL-DENY": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
+    "COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
+    "COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
+    "LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
+    "LOG-CONTENT-OFF": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
+    "CALLBACK-SUCCESS": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
+    "CALLBACK-FAILURE": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
+  }
+}

.github/pull_request_template.md

@@ -1,10 +1,13 @@
 <!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
      everyday engineering language, extremely parsable and readable at a glance. This goes double for
-     the TLDR, User Flow, and Caveats sections -->
+     the TLDR, User Flow, and Caveats sections
+     Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
+     "## Affected release" with nothing under it must not appear in the final description -->
 
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
+     If the PR intentionally changes what existing users see or how a screen behaves, add a line under the bullets that starts "Intentional product change:" describing what changes, why, and what users lose. Reviewers must never have to infer a deliberate UX change from the diff -->
 
 Problem this solves:
 
@@ -21,11 +24,13 @@ How it solves it:
 <!-- Two ordered lists, Before and After, walking the same end user through the same task, written strictly from that user's seat
      Read the linked issue, ticket, or customer thread first so the flow reflects the real application and the routes its users actually hit; don't invent a generic scenario
      Lead each list with one plain sentence saying where the flow fails (Before) or succeeds (After), then number the steps
+     Keep it tight: aim for 3 to 5 steps per list, one line each, roughly 20 words max, and never pad a shorter flow with filler steps to hit the count. Cover the one path the PR changes and fold variants (case, other field, second endpoint) into a clause on the step they belong to rather than their own steps. The example below is the target length
      Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
      No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
      Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
      If the bug had a security or authorization consequence, end each list with what another user could or could no longer do
-     Regenerate this section whenever new commits change the PR's behavior, so it never describes an older revision
+     Regenerate this section, screenshots included, whenever new commits change the PR's behavior, so it never describes an older revision
+     If the PR changes what an Admin UI page shows, embed a before and an after screenshot of that page right after its list, taken at the same URL on the same data, with the rows, fields, or controls that changed boxed in red so a reader spots the difference without reading the steps. These are the UI screenshots for Screenshots / Proof of Fix too: embed them once here and have that section's Before and After steps point back to them instead of repeating the images
 
 Example:
 
@@ -45,22 +50,22 @@ After: the same request comes back with real token counts, so the dashboard show
 
 ## Relevant issues
 
-<!-- e.g., "Fixes #000" -->
+<!-- e.g., "Fixes #000". Drop the section if there is none -->
 
 ## Affected release
 
-<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Leave the section blank otherwise -->
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1". Add the `backport-stable` label only when the regression is a P0, meaning its Linear ticket is Urgent (a security hole however narrow, data loss, or a crash or outage for every user on that version), because every labeled PR must be cherry-picked onto the baking rc line before the stable can be tagged; every other regression fix ships in the next rc unlabeled. Drop the section otherwise -->
 
 ## Linear ticket
 
-<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
+<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, drop the section rather than guessing -->
 
 ## Pre-Submission checklist
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
 - [ ] I have added meaningful tests
-- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
+- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
 - [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
 - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
@@ -134,7 +139,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      human reader
      If you assumed something instead of testing it, e.g. "only reproduces with X on" or "no
      user-observable behavior difference", list it here too with what breaks if it is wrong
-     Leave this section empty if there are none -->
+     Drop this section if there are none -->
 
 ## QA runbook
 

.github/scripts/assert_ci_coverage.py

@@ -9,6 +9,7 @@ import sys
 import warnings
 from collections.abc import Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 import yaml
@@ -34,9 +35,8 @@ GLOB_CHARS = frozenset("*?")
 # tests has to be named by some shard or it runs nowhere. A child listed here is
 # itself decomposed one level deeper and is checked through its own entry.
 SHARDED_ROOTS: tuple[str, ...] = (
-    "tests/proxy_unit_tests",
     "tests/test_litellm",
-    "tests/test_litellm/proxy",
+    "tests/unit/proxy",
 )
 
 
@@ -120,6 +120,50 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
     )
 
 
+SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;")
+
+
+def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]:
+    script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
+    if not script.is_file():
+        return MappingProxyType({})
+    text: Final = _uncommented(script.read_text())
+    return MappingProxyType(
+        {
+            label: frozenset(
+                match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)
+            )
+            for label, body in SELECTION_ARM_RE.findall(text)
+        }
+    )
+
+
+def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
+    return frozenset(
+        token for tokens in _unit_selection_arms(repo_root).values() for token in tokens
+    )
+
+
+def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        scalar.value
+        for scalar in scalars
+        if scalar.key == "unit-flag" and "${{" not in scalar.value
+    )
+
+
+def _shard_tokens(
+    scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]
+) -> frozenset[str]:
+    wired: Final = _wired_unit_flags(scalars)
+    return _invoked_test_tokens(scalars) | frozenset(
+        token
+        for label, tokens in arms.items()
+        if label in wired
+        for token in tokens
+    )
+
+
 def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
     return frozenset(
         match.group(0)
@@ -235,9 +279,7 @@ class Slice:
             return True  # a `-k` this parser cannot model is assumed to claim everything
         if any(term.lower() in relative_path.lower() for term in self.excluded):
             return False
-        return not self.required or any(
-            term.lower() in name.lower() for term in self.required for name in inner_names
-        )
+        return not self.required or any(term.lower() in name.lower() for term in self.required for name in inner_names)
 
 
 def _strings(node: object) -> Iterable[str]:
@@ -307,9 +349,7 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
     except (OSError, SyntaxError):
         return frozenset({relative_path})
     return frozenset({relative_path}) | frozenset(
-        node.name
-        for node in ast.walk(tree)
-        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+        node.name for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
     )
 
 
@@ -331,9 +371,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
     slices: Final = _slices()
     named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
-        path
-        for path in _test_files()
-        if any(_token_covers(glob, path) for slice_ in slices for glob in slice_.globs)
+        path for path in _test_files() if any(_token_covers(glob, path) for slice_ in slices for glob in slice_.globs)
     )
     return tuple(
         Finding(
@@ -363,11 +401,7 @@ def _shard_children(root: str, repo_root: pathlib.Path = REPO_ROOT) -> tuple[str
             child.relative_to(repo_root).as_posix()
             for child in (repo_root / root).iterdir()
             if not child.name.startswith(".")
-            and (
-                _holds_tests(child)
-                if child.is_dir()
-                else child.name.startswith("test_") and child.suffix == ".py"
-            )
+            and (_holds_tests(child) if child.is_dir() else child.name.startswith("test_") and child.suffix == ".py")
         )
     )
 
@@ -484,7 +518,7 @@ def _check_slices() -> int:
 
 
 def _check_shards() -> int:
-    findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
+    findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms()))
     if findings:
         _report(
             "test directories and files that no shard claims",
@@ -499,13 +533,32 @@ def _check_shards() -> int:
     return 0
 
 
+def _integration_groups(runner: pathlib.Path) -> dict[str, tuple[str, ...]]:
+    module: Final = ast.parse(runner.read_text())
+    literal: Final = next(
+        node.value
+        for node in module.body
+        if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.target.id == "GROUPS"
+    )
+    mapping: Final = literal.args[0] if isinstance(literal, ast.Call) else literal
+    return {group: tuple(folders) for group, folders in ast.literal_eval(mapping).items()}
+
+
 def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
-    manifest: Final = repo_root / "tests/integration/contracts.json"
-    if not manifest.exists():
+    runner: Final = repo_root / "tests/integration/run.py"
+    if not runner.exists():
         return frozenset(), ()
-    entries: Final = json.loads(manifest.read_text())
-    paths: Final = frozenset(node.split("::", 1)[0] for node in entries["tests"])
-    browser_paths: Final = frozenset(node.split("::", 1)[0] for node in entries.get("browser", {}))
+    groups: Final = _integration_groups(runner)
+    integration_root: Final = repo_root / "tests/integration"
+    paths: Final = frozenset(
+        str(path.relative_to(repo_root))
+        for folders in groups.values()
+        for folder in folders
+        for path in (integration_root / folder).rglob("test_*.py")
+    )
+    browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
+    browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()
+    browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes)
     circle_path: Final = repo_root / ".circleci/config.yml"
     circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
     steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
@@ -526,15 +579,14 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
     )
     required: Final = (frozenset({"browser"}) if browser_paths else frozenset()) | frozenset(
         group
-        for group, folders in entries["groups"].items()
+        for group, folders in groups.items()
         if any(any(path.startswith(f"tests/integration/{folder}/") for folder in folders) for path in paths)
     )
     ungrouped: Final = frozenset(
         path
         for path in paths
         if sum(
-            any(path.startswith(f"tests/integration/{folder}/") for folder in folders)
-            for folders in entries["groups"].values()
+            any(path.startswith(f"tests/integration/{folder}/") for folder in folders) for folders in groups.values()
         )
         != 1
     )
@@ -547,10 +599,6 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         Finding(path, "integration contract is also selected by GitHub Actions")
         for path in paths
         if any(_token_covers(token, path) for token in gha_tokens)
-    ) + tuple(
-        Finding(path, "canonical integration test file is missing")
-        for path in paths
-        if not (repo_root / path).is_file()
     )
     browser_commands: Final = tuple(
         scalar.value
@@ -592,7 +640,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
     ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped))
     if not paths or not invoked or not scheduled:
         return frozenset(), findings + (
-            Finding(str(manifest.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
+            Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
         )
     return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings
 
@@ -607,7 +655,10 @@ def main() -> int:
     scalars = _all_scalars()
 
     integration_paths, ownership_findings = _integration_ownership()
-    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings
+    test_findings = (
+        _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
+        + ownership_findings
+    )
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
 

.github/scripts/read_rc_version.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Print `version=X.Y.0` from [project].version in pyproject.toml for $GITHUB_OUTPUT.
+
+Usage
+-----
+    python3 read_rc_version.py [path/to/pyproject.toml] >> "$GITHUB_OUTPUT"
+
+Exit code 1 with a `::error::` line on stderr when the version is not an X.Y.0 release.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+from typing import Final
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+RELEASE_VERSION: Final = re.compile(r"[0-9]+\.[0-9]+\.0")
+
+
+def read_version(pyproject: pathlib.Path) -> str:
+    with pyproject.open("rb") as f:
+        return tomllib.load(f)["project"]["version"]
+
+
+def main(argv: list[str]) -> int:
+    pyproject: Final = pathlib.Path(argv[1]) if len(argv) > 1 else pathlib.Path("pyproject.toml")
+    version: Final = read_version(pyproject)
+    if RELEASE_VERSION.fullmatch(version) is None:
+        print(  # noqa: T201  # the ::error:: line to stderr is the workflow's failure signal
+            f"::error::pyproject.toml version {version} is not an X.Y.0 release version", file=sys.stderr
+        )
+        return 1
+    print(f"version={version}")  # noqa: T201  # stdout line is appended to $GITHUB_OUTPUT
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/scripts/run_merge_smoke.py

@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+"""Merge smoke harness: bounded checks run inside a loopback-only Linux network namespace."""
+
+# ruff: noqa: T201  # CLI harness: stdout/stderr lines are the reported result
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import http.client
+import json
+import os
+import secrets
+import signal
+import socket
+import subprocess
+import sys
+import time
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, NoReturn, TextIO, cast
+
+import pytest
+
+EXPECTED_CASES: Final = (
+    "CHAT-JSON",
+    "CHAT-TEXT-STREAM",
+    "CHAT-TOOL-STREAM",
+    "MODEL-ALLOW",
+    "MODEL-DENY",
+    "COST-EXPLICIT",
+    "COST-ZERO",
+    "LOG-CONTENT-ON",
+    "LOG-CONTENT-OFF",
+    "CALLBACK-SUCCESS",
+    "CALLBACK-FAILURE",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class CheckResult:
+    ok: bool
+    detail: str = ""
+
+
+@dataclass(slots=True)
+class _Args:
+    command: str = ""
+    no_child: bool = False
+    expect: str = ""
+    litellm_bin: str | None = None
+    lite_bin: str | None = None
+    diagnostics_dir: str = ""
+    ready_deadline: float = 120.0
+    shutdown_deadline: float = 20.0
+    poll_interval: float = 0.5
+    manifest: str = ""
+    rootdir: str | None = None
+
+
+def fail(reason: str) -> NoReturn:
+    print(f"merge-smoke: FAIL {reason}", file=sys.stderr)
+    sys.exit(1)
+
+
+def ok(step: str) -> None:
+    print(f"merge-smoke: OK {step}")
+
+
+def tail(path: Path, lines: int = 20) -> str:
+    try:
+        return "\n".join(path.read_text(errors="replace").splitlines()[-lines:])
+    except OSError as exc:
+        return f"<cannot read {path}: {exc}>"
+
+
+def cmd_verify_isolation(args: _Args) -> int:
+    if os.geteuid() == 0:
+        fail("verify-isolation must run unprivileged (geteuid()==0)")
+    try:
+        socket.create_connection(("192.0.2.1", 9), timeout=3)
+    except OSError as exc:
+        print(f"external connect blocked as expected: errno={exc.errno} {exc}")
+    else:
+        fail("external TCP connect to 192.0.2.1:9 succeeded; namespace is not isolated")
+    listener: Final = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.bind(("127.0.0.1", 0))
+    listener.listen(1)
+    port: Final = cast(int, listener.getsockname()[1])
+    client: Final = socket.create_connection(("127.0.0.1", port), timeout=5)
+    accepted: Final = listener.accept()
+    accepted[0].close()
+    client.close()
+    listener.close()
+    print(f"loopback connect ok on 127.0.0.1:{port}")
+    if not args.no_child:
+        proc: Final = subprocess.run(
+            [sys.executable, str(Path(__file__).resolve()), "verify-isolation", "--no-child"],
+            timeout=30,
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            fail(f"child process did not inherit isolation: {proc.stderr.strip()}")
+        print("child process inherits isolation")
+    ok("verify-isolation")
+    return 0
+
+
+def cmd_interpreter(args: _Args) -> int:
+    print(sys.version)
+    print(sys.executable)
+    actual: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
+    if actual != args.expect:
+        fail(f"interpreter is {actual}, expected {args.expect}")
+    ok(f"interpreter {actual}")
+    return 0
+
+
+def _run_cli(argv: Sequence[str], label: str) -> CheckResult:
+    try:
+        proc: Final = subprocess.run(list(argv), timeout=120, capture_output=True, text=True)
+    except subprocess.TimeoutExpired:
+        return CheckResult(ok=False, detail=f"{label} timed out after 120s")
+    sys.stdout.write(proc.stdout)
+    sys.stderr.write(proc.stderr)
+    if proc.returncode != 0:
+        return CheckResult(ok=False, detail=f"{label} exited {proc.returncode}")
+    return CheckResult(ok=True)
+
+
+def cmd_cli(args: _Args) -> int:
+    venv_bin: Final = Path(sys.executable).parent
+    litellm_bin: Final = Path(args.litellm_bin) if args.litellm_bin else venv_bin / "litellm"
+    lite_bin: Final = Path(args.lite_bin) if args.lite_bin else venv_bin / "lite"
+    commands: Final = (
+        ("import litellm", [sys.executable, "-c", "import litellm"]),
+        ("litellm --version", [str(litellm_bin), "--version"]),
+        ("lite version", [str(lite_bin), "version"]),
+    )
+    for label, argv in commands:
+        result = _run_cli(argv, label)
+        if not result.ok:
+            fail(result.detail)
+        ok(label)
+    return 0
+
+
+def _free_port() -> int:
+    sock: Final = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind(("127.0.0.1", 0))
+    port: Final = cast(int, sock.getsockname()[1])
+    sock.close()
+    return port
+
+
+_CONFIG_TEMPLATE: Final = """model_list:
+  - model_name: smoke-model
+    litellm_params:
+      model: openai/smoke-model
+      api_base: http://127.0.0.1:9/v1
+      api_key: synthetic-key
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+"""
+
+
+def _listen_inode(port: int) -> str | None:
+    target: Final = f"{port:04X}"
+    for table in ("/proc/net/tcp", "/proc/net/tcp6"):
+        try:
+            rows = Path(table).read_text().splitlines()[1:]
+        except OSError:
+            continue
+        for row in rows:
+            cols = row.split()
+            if len(cols) > 9 and cols[3] == "0A" and cols[1].rsplit(":", 1)[-1] == target:
+                return cols[9]
+    return None
+
+
+def _ancestors(pid: int) -> frozenset[int]:
+    chain: Final[set[int]] = set()
+    pending: Final[list[int]] = [pid]
+    while pending:
+        current = pending.pop()
+        if current <= 0 or current in chain:
+            continue
+        chain.add(current)
+        try:
+            stat = Path(f"/proc/{current}/stat").read_text()
+        except OSError:
+            continue
+        pending.append(int(stat.rpartition(")")[2].split()[1]))
+    return frozenset(chain)
+
+
+def _socket_owner_pid(inode: str) -> int | None:
+    for proc_dir in Path("/proc").iterdir():
+        if not proc_dir.name.isdigit():
+            continue
+        fd_dir = proc_dir / "fd"
+        try:
+            for fd in fd_dir.iterdir():
+                try:
+                    if os.readlink(fd) == f"socket:[{inode}]":
+                        return int(proc_dir.name)
+                except OSError:
+                    continue
+        except OSError:
+            continue
+    return None
+
+
+def _verify_port_owner(port: int, proc: subprocess.Popen[bytes]) -> CheckResult:
+    inode: Final = _listen_inode(port)
+    if inode is None:
+        return CheckResult(ok=False, detail=f"no LISTEN socket found for port {port} in /proc/net/tcp")
+    owner: Final = _socket_owner_pid(inode)
+    if owner is None:
+        return CheckResult(ok=False, detail=f"no process owns the listen socket inode {inode} for port {port}")
+    if owner != proc.pid and proc.pid not in _ancestors(owner):
+        return CheckResult(
+            ok=False, detail=f"port {port} owned by pid {owner} outside the launched process group {proc.pid}"
+        )
+    if proc.poll() is not None:
+        return CheckResult(ok=False, detail=f"proxy exited with code {proc.returncode} after readiness")
+    return CheckResult(ok=True)
+
+
+def cmd_proxy_startup(args: _Args) -> int:
+    diagnostics: Final = Path(args.diagnostics_dir)
+    diagnostics.mkdir(parents=True, exist_ok=True)
+    venv_bin: Final = Path(sys.executable).parent
+    litellm_bin: Final = Path(args.litellm_bin) if args.litellm_bin else venv_bin / "litellm"
+    port: Final = _free_port()
+    master_key: Final = "sk-smoke-" + secrets.token_hex(16)
+    config_path: Final = diagnostics / "config.yaml"
+    config_path.write_text(_CONFIG_TEMPLATE)
+    log_path: Final = diagnostics / "proxy.log"
+    result_path: Final = diagnostics / "result.json"
+    outcome: Final[dict[str, object]] = {
+        "port": port,
+        "time_to_ready_s": None,
+        "shutdown_s": None,
+        "readiness": None,
+        "outcome": "failed",
+    }
+    log_file: Final = log_path.open("w")
+    env: Final = {
+        **os.environ,
+        "LITELLM_MASTER_KEY": master_key,
+        "LITELLM_LOCAL_MODEL_COST_MAP": "True",
+    }
+    started: Final = time.monotonic()
+    proc: Final = subprocess.Popen(
+        [str(litellm_bin), "--config", str(config_path), "--host", "127.0.0.1", "--port", str(port)],
+        stdout=log_file,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,
+        env=env,
+    )
+    body: str | None = None
+    last_status: int | None = None
+    while time.monotonic() - started < args.ready_deadline:
+        if proc.poll() is not None:
+            log_file.close()
+            result_path.write_text(json.dumps(outcome))
+            fail(f"proxy exited early with code {proc.returncode}\n{tail(log_path)}")
+        try:
+            conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
+            conn.request("GET", "/health/readiness")
+            resp = conn.getresponse()
+            last_status = resp.status
+            candidate = resp.read().decode()
+            conn.close()
+        except (http.client.HTTPException, ConnectionError, OSError):
+            time.sleep(args.poll_interval)
+            continue
+        if last_status == 200:
+            body = candidate
+            break
+        time.sleep(args.poll_interval)
+    outcome["time_to_ready_s"] = round(time.monotonic() - started, 3)
+    if body is None:
+        _terminate(proc, log_file)
+        result_path.write_text(json.dumps(outcome))
+        detail = f"last status {last_status}" if last_status is not None else "no response"
+        fail(f"readiness not reached within {args.ready_deadline}s ({detail})\n{tail(log_path)}")
+    outcome["readiness"] = body
+    try:
+        readiness = cast(object, json.loads(body))
+    except json.JSONDecodeError:
+        readiness = None
+    if readiness != {"status": "healthy", "db": "Not connected"}:
+        _terminate(proc, log_file)
+        result_path.write_text(json.dumps(outcome))
+        fail(f"unexpected readiness body: {body}")
+    owner_check: Final = _verify_port_owner(port, proc)
+    if not owner_check.ok:
+        _terminate(proc, log_file)
+        result_path.write_text(json.dumps(outcome))
+        fail(owner_check.detail)
+    shutdown_started: Final = time.monotonic()
+    os.killpg(proc.pid, signal.SIGTERM)
+    try:
+        proc.wait(timeout=args.shutdown_deadline)
+    except subprocess.TimeoutExpired:
+        os.killpg(proc.pid, signal.SIGKILL)
+        proc.wait(timeout=10)
+        outcome["shutdown_s"] = round(time.monotonic() - shutdown_started, 3)
+        log_file.close()
+        result_path.write_text(json.dumps(outcome))
+        fail(f"forced kill after {args.shutdown_deadline}s\n{tail(log_path)}")
+    outcome["shutdown_s"] = round(time.monotonic() - shutdown_started, 3)
+    try:
+        os.killpg(proc.pid, 0)
+    except ProcessLookupError:
+        pass
+    else:
+        os.killpg(proc.pid, signal.SIGKILL)
+        log_file.close()
+        result_path.write_text(json.dumps(outcome))
+        fail("process group survived SIGTERM")
+    log_file.close()
+    outcome["outcome"] = "ok"
+    result_path.write_text(json.dumps(outcome))
+    ok(f"proxy-startup ready={outcome['time_to_ready_s']}s shutdown={outcome['shutdown_s']}s")
+    return 0
+
+
+def _terminate(proc: subprocess.Popen[bytes], log_file: TextIO) -> None:
+    with contextlib.suppress(ProcessLookupError):
+        os.killpg(proc.pid, signal.SIGTERM)
+    try:
+        proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        with contextlib.suppress(ProcessLookupError):
+            os.killpg(proc.pid, signal.SIGKILL)
+        with contextlib.suppress(subprocess.TimeoutExpired):
+            proc.wait(timeout=10)
+    log_file.close()
+
+
+def _load_manifest(path: Path) -> MappingProxyType[str, str]:
+    def no_duplicates(pairs: list[tuple[object, object]]) -> dict[object, object]:
+        seen: dict[object, object] = {}
+        for key, value in pairs:
+            if key in seen:
+                raise ValueError(f"duplicate key in manifest: {key}")
+            seen[key] = value
+        return seen
+
+    raw_value: object = cast(object, json.loads(path.read_text(), object_pairs_hook=no_duplicates))
+    if not isinstance(raw_value, dict):
+        raise ValueError("manifest must be an object")
+    loaded: Final = cast(dict[object, object], raw_value)
+    cases_value: object = loaded.get("cases")
+    if not isinstance(cases_value, dict):
+        raise ValueError("manifest must be an object with a 'cases' object")
+    cases_any: Final = cast(dict[object, object], cases_value)
+    cases: Final = {k: v for k, v in cases_any.items() if isinstance(k, str) and isinstance(v, str)}
+    if len(cases) != len(cases_any):
+        raise ValueError("manifest 'cases' must map string ids to string node ids")
+    return MappingProxyType(cases)
+
+
+@dataclass(slots=True, eq=False)
+class _Recorder:
+    collect_failed: list[str] = field(default_factory=list)
+    collected: tuple[str, ...] = ()
+    reports: dict[str, list[tuple[str, str, bool]]] = field(default_factory=dict)
+
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+        if report.failed:
+            self.collect_failed.append(report.nodeid)
+
+    def pytest_collection_finish(self, session: pytest.Session) -> None:
+        self.collected = tuple(item.nodeid for item in session.items)
+
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+        self.reports.setdefault(report.nodeid, []).append((report.when, report.outcome, hasattr(report, "wasxfail")))
+
+
+def cmd_pytest(args: _Args) -> int:
+    try:
+        cases: Final = _load_manifest(Path(args.manifest))
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        fail(f"manifest invalid: {exc}")
+    if tuple(cases) != EXPECTED_CASES:
+        fail(f"manifest case ids must be exactly {list(EXPECTED_CASES)} in order, got {list(cases)}")
+    node_ids: Final = tuple(cases.values())
+    if len(set(node_ids)) != len(node_ids):
+        fail("manifest node ids are not unique")
+    argv: Final = [
+        *node_ids,
+        "-p",
+        "no:cacheprovider",
+        "-p",
+        "no:xdist",
+        "-p",
+        "no:rerunfailures",
+        "-p",
+        "no:randomly",
+        "-rA",
+        "-q",
+        *(["--rootdir", args.rootdir] if args.rootdir else []),
+    ]
+
+    recorder: Final = _Recorder()
+    code: Final = pytest.main(argv, plugins=[recorder])
+    name_of: Final = MappingProxyType({node_id: case_id for case_id, node_id in cases.items()})
+    problems: Final[list[str]] = []
+    if code != 0:
+        problems.append(f"pytest exit code {code}")
+    for failed_id in recorder.collect_failed:
+        problems.append(f"collection failed: {name_of.get(failed_id, failed_id)}")
+    expected: Final = Counter(node_ids)
+    collected: Final = Counter(recorder.collected)
+    for node_id in expected - collected:
+        problems.append(f"missing case {name_of[node_id]} ({node_id})")
+    for node_id in collected - expected:
+        problems.append(f"unexpected test collected: {node_id}")
+    for node_id, count in collected.items():
+        if count > 1:
+            problems.append(f"duplicated test id: {node_id}")
+    if len(recorder.collected) != len(EXPECTED_CASES):
+        problems.append(f"collected {len(recorder.collected)} tests, expected {len(EXPECTED_CASES)}")
+    rows: Final[list[tuple[str, bool]]] = []
+    for case_id, node_id in cases.items():
+        reports = recorder.reports.get(node_id, [])
+        case_ok = (
+            bool(reports)
+            and all(outcome == "passed" and not wasxfail for _, outcome, wasxfail in reports)
+            and {when for when, _, _ in reports} >= {"setup", "call", "teardown"}
+        )
+        rows.append((case_id, case_ok))
+        if not reports:
+            problems.append(f"{case_id} ({node_id}) produced no runtest reports")
+            continue
+        for when, outcome, wasxfail in reports:
+            if outcome != "passed":
+                problems.append(f"{case_id} ({node_id}) {when} outcome={outcome}")
+            if wasxfail:
+                problems.append(f"{case_id} ({node_id}) {when} was xfail/xpass")
+        missing_phases = {"setup", "call", "teardown"} - {when for when, _, _ in reports}
+        for phase in sorted(missing_phases):
+            problems.append(f"{case_id} ({node_id}) missing {phase} report")
+    for case_id, passed in rows:
+        print(f"{case_id}  {'PASS' if passed else 'FAIL'}  {cases[case_id]}")
+    if problems:
+        for problem in problems:
+            print(f"merge-smoke: {problem}", file=sys.stderr)
+        fail("pytest verdict failed")
+    ok("pytest 11 cases")
+    return 0
+
+
+def main() -> int:
+    parser: Final = argparse.ArgumentParser(description=__doc__)
+    subs: Final = parser.add_subparsers(dest="command", required=True)
+    p_iso: Final = subs.add_parser("verify-isolation")
+    p_iso.add_argument("--no-child", action="store_true")
+    p_interp: Final = subs.add_parser("interpreter")
+    p_interp.add_argument("--expect", required=True)
+    p_cli: Final = subs.add_parser("cli")
+    p_cli.add_argument("--litellm-bin", default=None)
+    p_cli.add_argument("--lite-bin", default=None)
+    p_proxy: Final = subs.add_parser("proxy-startup")
+    p_proxy.add_argument("--diagnostics-dir", required=True)
+    p_proxy.add_argument("--litellm-bin", default=None)
+    p_proxy.add_argument("--ready-deadline", type=float, default=120)
+    p_proxy.add_argument("--shutdown-deadline", type=float, default=20)
+    p_proxy.add_argument("--poll-interval", type=float, default=0.5)
+    p_test: Final = subs.add_parser("pytest")
+    p_test.add_argument("--manifest", required=True)
+    p_test.add_argument("--rootdir", default=None)
+    args: Final = parser.parse_args(namespace=_Args())
+    handlers: Final = {
+        "verify-isolation": cmd_verify_isolation,
+        "interpreter": cmd_interpreter,
+        "cli": cmd_cli,
+        "proxy-startup": cmd_proxy_startup,
+        "pytest": cmd_pytest,
+    }
+    return handlers[args.command](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/verify_linux_native_wheel.py

@@ -214,7 +214,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 35_000_000
+    native_size_limit: Final = 45_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),

.github/workflows/_test-unit-base.yml

@@ -13,6 +13,16 @@ on:
           have its path existence-checked like any other token.
         required: true
         type: string
+      unit-flag:
+        description: >-
+          Codecov flag of the `.circleci/tests.yml` job that now owns part of
+          this shard. The shard also runs the files
+          `.circleci/scripts/unit_selection.sh` lists for the flag, on every
+          event, because the CircleCI pipeline is manual-only while the tests
+          migrate.
+        required: false
+        type: string
+        default: ""
       workers:
         description: "Number of pytest-xdist workers"
         required: false
@@ -92,6 +102,7 @@ jobs:
       pull-requests: read
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
+      has-coverage: ${{ steps.tests.outputs.has-coverage }}
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -121,12 +132,7 @@ jobs:
       - name: Cache uv dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 5
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
+        uses: ./.github/actions/cache-uv-downloads
 
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
@@ -160,10 +166,12 @@ jobs:
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
+        id: tests
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
+          UNIT_FLAG: ${{ inputs.unit-flag }}
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
@@ -171,9 +179,18 @@ jobs:
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
+          echo "has-coverage=false" >> "$GITHUB_OUTPUT"
+          selection="${TEST_PATH}"
+          if [ -n "${UNIT_FLAG}" ]; then
+            selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')"
+          fi
+          if [ -z "${selection// /}" ]; then
+            echo "shard selection is empty; nothing to run"
+            exit 0
+          fi
           pytest_args=()
           existing_paths=0
-          for token in ${TEST_PATH:?}; do
+          for token in ${selection}; do
             case "${token}" in
               -*) pytest_args+=("${token}") ;;
               *)
@@ -187,7 +204,7 @@ jobs:
             esac
           done
           if [ "${existing_paths}" -eq 0 ]; then
-            echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
+            echo "No path in the selection exists (${selection}); nothing to run"
             exit 0
           fi
           xdist_args=()
@@ -209,8 +226,11 @@ jobs:
             --cov-config=pyproject.toml
           status=$?
           set -e
+          if [ -f coverage.xml ]; then
+            echo "has-coverage=true" >> "$GITHUB_OUTPUT"
+          fi
           if [ "$status" -eq 5 ]; then
-            echo "pytest collected no tests from ${TEST_PATH}; passing"
+            echo "pytest collected no tests from ${selection}; passing"
             exit 0
           fi
           exit "$status"
@@ -226,7 +246,7 @@ jobs:
   upload-coverage:
     name: Upload coverage to Codecov
     needs: run
-    if: always() && needs.run.outputs.decision != 'skip'
+    if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -249,7 +269,7 @@ jobs:
       - name: Upload to Codecov
         id: codecov-upload
         continue-on-error: true
-        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
         with:
           use_oidc: true
           directory: coverage-reports
@@ -260,7 +280,7 @@ jobs:
       - name: Upload to Codecov (retry)
         if: steps.codecov-upload.outcome == 'failure'
         continue-on-error: true
-        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
         with:
           use_oidc: true
           directory: coverage-reports

.github/workflows/check-ui-api-types.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 permissions:

.github/workflows/ci-coverage.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
 
 permissions:
   contents: read

.github/workflows/codeql.yml

@@ -67,12 +67,18 @@ jobs:
       # further up the stack are modified. The suppression is scoped to this one
       # file/rule pair via SARIF post-filtering so every other callsite of
       # py/weak-sensitive-data-hashing in the repository continues to be analyzed.
-      - name: Filter SARIF (OCI sha256)
+      # The same query fires on the HIBP k-anonymity lookup in
+      # litellm/proxy/auth/password_policy.py, where the password's SHA-1 is only
+      # a lookup key into the haveibeenpwned range API (the protocol mandates
+      # SHA-1) and the digest itself never leaves the proxy beyond its first 5
+      # characters.
+      - name: Filter SARIF (OCI sha256, HIBP sha1)
         if: matrix.language == 'python'
         uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
         with:
           patterns: |
             -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
+            -litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing
           input: sarif-results/python.sarif
           output: sarif-results/python.sarif
 

.github/workflows/codspeed.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - litellm_internal_staging
     paths:
       - "litellm/**"
       - "tests/benchmarks/**"
@@ -17,7 +16,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
     paths:
       - "litellm/**"
       - "tests/benchmarks/**"

.github/workflows/compat-matrix-image.yml

@@ -0,0 +1,42 @@
+name: Compat Matrix Image
+
+on:
+  pull_request:
+    paths:
+      - tests/e2e/claude_code/cron_vm/**
+      - tests/e2e/claude_code/pr_gate_version_resolver.py
+      - .github/workflows/compat-matrix-image.yml
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  compat-matrix-image:
+    name: compat-matrix-image
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build the Render cron image
+        run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
+
+      - name: Resolve and install the Claude Code CLI as the cron user
+        run: |
+          docker run --rm compat-matrix:${{ github.sha }} bash -c '
+            set -euo pipefail
+            whoami
+            gh --version
+            uv --version
+            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
+            /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
+            /tmp/claude-cli/claude --version
+          '

.github/workflows/conventional-commits.yml

@@ -41,6 +41,7 @@ jobs:
             ci
             chore
             revert
+            security
           requireScope: false
           subjectPattern: ^(?![A-Z]).+$
           subjectPatternError: |

.github/workflows/cost-map-guard.yml

@@ -4,8 +4,6 @@ on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the P
   pull_request_target:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 permissions:

.github/workflows/create-rc-branch.yml

@@ -0,0 +1,79 @@
+name: Create RC Branch
+
+on:
+  schedule:
+    - cron: "0 3 * * 5"
+      timezone: "America/Los_Angeles"
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  create-rc-branch:
+    name: Create RC Branch
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Require main
+        env:
+          REF: ${{ github.ref }}
+        run: |
+          if [ "$REF" != "refs/heads/main" ]; then
+            echo "::error::rc branches are cut from refs/heads/main only, got $REF"
+            exit 1
+          fi
+
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Read release version
+        id: version
+        run: python3 .github/scripts/read_rc_version.py >> "$GITHUB_OUTPUT"
+
+      - name: Create rc branch
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const branchName = `rc/${process.env.VERSION}`;
+            const ref = `heads/${branchName}`;
+
+            const existing = await github.rest.git.getRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref,
+            }).catch((error) => {
+              if (error.status === 404) {
+                return null;
+              }
+              throw error;
+            });
+            if (existing !== null) {
+              core.setFailed(`Branch ${branchName} already exists at ${existing.data.object.sha}; leaving it untouched`);
+              return;
+            }
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/${ref}`,
+              sha: context.sha,
+            });
+            core.info(`Created branch ${branchName} at ${context.sha}`);
+
+  linear-release:
+    name: Move the Linear release to rc
+    needs: create-rc-branch
+    permissions:
+      contents: read
+    uses: ./.github/workflows/linear-release.yml
+    with:
+      rc_version: ${{ needs.create-rc-branch.outputs.version }}
+    secrets:
+      LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}

.github/workflows/create-release.yml

@@ -1,186 +0,0 @@
-name: Create Release
-
-on:
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
-        required: true
-        type: string
-      commit_hash:
-        description: "Full 40-char commit SHA to target"
-        required: true
-        type: string
-
-permissions: {}
-
-jobs:
-  release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Validate inputs
-        env:
-          TAG: ${{ inputs.tag }}
-          COMMIT_HASH: ${{ inputs.commit_hash }}
-        run: |
-          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
-            echo "::error::commit_hash must be a full 40-character commit SHA"
-            exit 1
-          fi
-          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
-            exit 1
-          fi
-
-      - name: Create release
-        env:
-          TAG: ${{ inputs.tag }}
-          COMMIT_HASH: ${{ inputs.commit_hash }}
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
-        with:
-          script: |
-            const tag = process.env.TAG;
-            const commitHash = process.env.COMMIT_HASH;
-
-            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
-            // Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
-            // like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
-            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
-            // are stable maintenance releases, not pre-releases.
-            const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
-
-            // A stable release should only claim the repo "latest" badge when its
-            // version is >= the current latest. Otherwise a backport (e.g. 1.84.6)
-            // would steal "latest" from a newer line (e.g. 1.88.1).
-            const versionKey = (rawTag) => {
-              const m = String(rawTag).match(/^v?(\d+)\.(\d+)\.(\d+)/);
-              if (!m) return null;
-              const maintenance = String(rawTag).match(/(?:\.post|\.patch\.)(\d+)/i);
-              return [Number(m[1]), Number(m[2]), Number(m[3]), maintenance ? Number(maintenance[1]) : 0];
-            };
-            const isAtLeast = (a, b) => {
-              for (let i = 0; i < a.length; i++) {
-                if (a[i] !== b[i]) return a[i] > b[i];
-              }
-              return true;
-            };
-
-            const cosignSection = [
-              `## Verify Docker Image Signature`,
-              ``,
-              `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`,
-              ``,
-              `**Verify using the pinned commit hash (recommended):**`,
-              ``,
-              `A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`,
-              ``,
-              '```bash',
-              `cosign verify \\`,
-              `  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`,
-              `  ghcr.io/berriai/litellm:${tag}`,
-              '```',
-              ``,
-              `**Verify using the release tag (convenience):**`,
-              ``,
-              `Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`,
-              ``,
-              '```bash',
-              `cosign verify \\`,
-              `  --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`,
-              `  ghcr.io/berriai/litellm:${tag}`,
-              '```',
-              ``,
-              `Expected output:`,
-              ``,
-              '```',
-              `The following checks were performed on each of these signatures:`,
-              `  - The cosign claims were validated`,
-              `  - The signatures were verified against the specified public key`,
-              '```',
-              ``,
-              `---`,
-              ``,
-            ].join('\n');
-
-            try {
-              let makeLatest = "false";
-              const newVersion = versionKey(tag);
-              if (!isPrerelease && newVersion) {
-                let latestVersion = null;
-                try {
-                  const latest = await github.rest.repos.getLatestRelease({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                  });
-                  latestVersion = versionKey(latest.data.tag_name);
-                } catch (error) {
-                  if (error.status !== 404) throw error;
-                }
-                makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
-              }
-
-              try {
-                await github.rest.git.createRef({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: `refs/tags/${tag}`,
-                  sha: commitHash,
-                });
-              } catch (error) {
-                if (error.status !== 422) throw error;
-                const existing = await github.rest.git.getRef({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: `tags/${tag}`,
-                });
-                if (existing.data.object.sha !== commitHash) {
-                  throw new Error(`Tag ${tag} already exists at ${existing.data.object.sha}, expected ${commitHash}`);
-                }
-              }
-
-              const response = await github.rest.repos.createRelease({
-                draft: true,
-                generate_release_notes: true,
-                name: tag,
-                owner: context.repo.owner,
-                prerelease: isPrerelease,
-                repo: context.repo.repo,
-                tag_name: tag,
-              });
-
-              const updatedBody = cosignSection + (response.data.body ?? '');
-              await github.rest.repos.updateRelease({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                release_id: response.data.id,
-                tag_name: tag,
-                body: updatedBody,
-                draft: false,
-              });
-
-              if (!isPrerelease) {
-                await github.rest.repos.updateRelease({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  release_id: response.data.id,
-                  tag_name: tag,
-                  make_latest: makeLatest,
-                });
-              }
-
-            } catch (error) {
-              core.setFailed(error.message);
-            }
-
-  create-branch:
-    name: Create Release Branch
-    needs: release
-    permissions:
-      contents: write
-    uses: ./.github/workflows/create-release-branch.yml
-    with:
-      tag: ${{ inputs.tag }}
-      commit_hash: ${{ inputs.commit_hash }}

.github/workflows/create_daily_staging_branch.yml

@@ -1,49 +0,0 @@
-name: Create Daily Staging Branch
-
-on:
-  schedule:
-    - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC
-  workflow_dispatch: # Allow manual trigger
-
-jobs:
-  create-staging-branch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-
-    steps:
-      - name: Create daily staging branch
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
-          echo "Creating branch: $BRANCH_NAME"
-          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
-            echo "Branch $BRANCH_NAME already exists. Skipping creation."
-            exit 0
-          fi
-          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
-          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
-          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"
-
-  create-internal-dev-branch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-
-    steps:
-      - name: Create internal dev branch
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
-          echo "Creating branch: $BRANCH_NAME"
-          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
-            echo "Branch $BRANCH_NAME already exists. Skipping creation."
-            exit 0
-          fi
-          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
-          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
-          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/guard-fork-dependencies.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "uv.lock"

.github/workflows/image-scan.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
       - litellm_oss_branch
       - "litellm_**"
     paths:

.github/workflows/lens-worker.yml

@@ -0,0 +1,66 @@
+name: Lens Worker Image
+
+on:
+  pull_request:
+    branches: [main, litellm_oss_branch, "litellm_**"]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/lens/**
+      - .github/workflows/lens-worker.yml
+  push:
+    branches: [main]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/lens/**
+      - .github/workflows/lens-worker.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lens-worker-image:
+    permissions:
+      contents: read
+      packages: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Build Lens worker
+        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Verify standalone imports with a read-only filesystem
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
+            --security-opt no-new-privileges --entrypoint python \
+            lens-worker:${{ github.sha }} -c '
+          import os
+          import lens.worker
+          from lens.trace_store import trace_store
+          assert os.getuid() == 65532
+          with trace_store() as store:
+              assert store.count() == 0
+          '
+      - name: Verify recovery after temporary storage fills
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
+            -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
+            --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
+      - name: Publish versioned Lens worker
+        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+        env:
+          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }}
+        run: |
+          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
+          docker tag lens-worker:${{ github.sha }} "$IMAGE"
+          docker push "$IMAGE"
+          printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/linear-release.yml

@@ -0,0 +1,131 @@
+name: Linear Release
+
+on:
+  push:
+    branches:
+      - main
+      - "rc/**"
+  release:
+    types: [published]
+  workflow_call:
+    inputs:
+      rc_version:
+        description: "X.Y.0 release whose rc branch was just cut"
+        required: true
+        type: string
+    secrets:
+      LINEAR_API_KEY:
+        required: true
+
+permissions: {}
+
+jobs:
+  linear-release:
+    name: Linear Release
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Plan
+        id: plan
+        env:
+          EVENT: ${{ github.event_name }}
+          REF_NAME: ${{ github.ref_name }}
+          BEFORE: ${{ github.event.before }}
+          CREATED: ${{ github.event.created }}
+          RC_VERSION: ${{ inputs.rc_version }}
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+          PRERELEASE: ${{ github.event.release.prerelease }}
+        run: |
+          set -euo pipefail
+          sync_base="${BEFORE}"
+          if [ "${CREATED}" = "true" ]; then
+            sync_base=""
+          fi
+          if [ -n "${RC_VERSION}" ]; then
+            echo "version=${RC_VERSION}" >> "$GITHUB_OUTPUT"
+            echo "stage=rc" >> "$GITHUB_OUTPUT"
+          elif [ "${EVENT}" = "release" ]; then
+            if [ "${PRERELEASE}" = "true" ] || ! echo "${RELEASE_TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.0$'; then
+              echo "::notice::${RELEASE_TAG} is not an X.Y.0 stable release; nothing to complete"
+              exit 0
+            fi
+            echo "version=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT"
+            echo "complete=true" >> "$GITHUB_OUTPUT"
+          elif [ "${REF_NAME}" = "main" ]; then
+            version="$(python3 .github/scripts/read_rc_version.py | cut -d= -f2)"
+            status=0
+            git ls-remote --exit-code --heads origin "rc/${version}" > /dev/null || status=$?
+            case "${status}" in
+              0)
+                IFS=. read -r major minor _ <<< "${version}"
+                version="${major}.$((minor + 1)).0"
+                ;;
+              2) ;;
+              *)
+                echo "::error::could not check whether rc/${version} exists (git ls-remote exit ${status})"
+                exit 1
+                ;;
+            esac
+            echo "version=${version}" >> "$GITHUB_OUTPUT"
+            echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT"
+            echo "main=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=${REF_NAME#rc/}" >> "$GITHUB_OUTPUT"
+            echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT"
+            echo "stage=rc" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Sync commits into the release
+        if: steps.plan.outputs.sync_base != ''
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: sync
+          name: LiteLLM ${{ steps.plan.outputs.version }}
+          version: ${{ steps.plan.outputs.version }}
+          base_ref: ${{ steps.plan.outputs.sync_base }}
+          cli_version: v0.18.0
+
+      - name: Keep the main stage unless the rc branch was cut during this run
+        id: main_stage
+        if: steps.plan.outputs.main == 'true'
+        env:
+          VERSION: ${{ steps.plan.outputs.version }}
+        run: |
+          set -euo pipefail
+          status=0
+          git ls-remote --exit-code --heads origin "rc/${VERSION}" > /dev/null || status=$?
+          case "${status}" in
+            0) echo "::notice::rc/${VERSION} was cut during this run; leaving the release in its rc stage" ;;
+            2) echo "stage=main" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::could not check whether rc/${VERSION} exists (git ls-remote exit ${status})"
+              exit 1
+              ;;
+          esac
+
+      - name: Move the release to its stage
+        if: steps.plan.outputs.stage != '' || steps.main_stage.outputs.stage != ''
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: update
+          stage: ${{ steps.plan.outputs.stage || steps.main_stage.outputs.stage }}
+          version: ${{ steps.plan.outputs.version }}
+          cli_version: v0.18.0
+
+      - name: Complete the release
+        if: steps.plan.outputs.complete == 'true'
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: complete
+          version: ${{ steps.plan.outputs.version }}
+          cli_version: v0.18.0

.github/workflows/mutation-test.yml

@@ -44,6 +44,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -53,6 +54,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 

.github/workflows/osv-scan.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   schedule:
     - cron: "23 6 * * *"

.github/workflows/publish-basedpyright-base-counts.yml

@@ -1,6 +1,6 @@
 name: Publish basedpyright base counts
 
-# Every commit on main or litellm_internal_staging can become a future merge-base.
+# Every commit on main can become a future merge-base.
 # Publishing its per-rule basedpyright counts as an artifact lets
 # scripts/type_check_gate.py download them in seconds instead of paying a
 # 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
@@ -11,7 +11,6 @@ on:
   push:
     branches:
       - main
-      - litellm_internal_staging
   workflow_dispatch:
     inputs:
       ref:

.github/workflows/test-code-quality.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -47,6 +44,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -56,6 +54,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 
@@ -83,6 +92,14 @@ jobs:
       - name: test_e2e_changed_gate
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
 
+      - name: test_e2e_metadata
+        env:
+          PYTHONPATH: tests/e2e
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
+
+      - name: Check merge smoke harness
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 
@@ -146,6 +163,9 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_unbounded_in_lists (fails on findings not in the baseline)
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 permissions:
@@ -182,6 +180,18 @@ jobs:
             echo "No changed tests/e2e Python files; skipping."
           fi
 
+      - name: Run the claude_code harness unit tests
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/claude_code/**/*.py' ':(glob)tests/e2e/*.py' tests/e2e/claude_code/cron_vm/install_claude_code.sh pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
+            echo "No changed claude_code harness files; skipping."
+            exit 0
+          fi
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
+          tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
+          PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures tests/e2e/claude_code/_*_unit_tests
+
       - name: Check for circular imports
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-litellm-ui-build.yml

@@ -7,8 +7,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 concurrency:

.github/workflows/test-litellm-ui-lint.yml

@@ -6,8 +6,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 concurrency:

.github/workflows/test-litellm-ui-unit.yml

@@ -7,13 +7,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/test-mcp-dependency-resolution.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 permissions:

.github/workflows/test-merge-smoke.yml

@@ -0,0 +1,95 @@
+name: Merge smoke checks
+
+on:
+  pull_request:
+    branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: merge-smoke-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  dashboard-build:
+    name: Dashboard build
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build the dashboard stage
+        run: docker build --target ui-builder -f Dockerfile .
+
+  core-checks:
+    name: Core checks (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    env:
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
+
+      - name: Create the loopback-only network namespace
+        run: |
+          sudo ip netns add smoke
+          sudo ip netns exec smoke ip link set lo up
+          cat > "${RUNNER_TEMP}/in-netns" <<'WRAP'
+          #!/usr/bin/env bash
+          set -euo pipefail
+          exec sudo --preserve-env=LITELLM_LOCAL_MODEL_COST_MAP ip netns exec smoke setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups -- env HOME="${HOME}" PATH="${PATH}" "$@"
+          WRAP
+          chmod +x "${RUNNER_TEMP}/in-netns"
+          echo "IN_NETNS=${RUNNER_TEMP}/in-netns" >> "${GITHUB_ENV}"
+
+      - name: Verify namespace isolation
+        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py verify-isolation
+
+      - name: Verify interpreter version
+        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py interpreter --expect ${{ matrix.python-version }}
+
+      - name: Import and CLI checks
+        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
+
+      - name: Proxy startup check
+        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
+
+      - name: Run curated smoke cases
+        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py pytest --manifest .github/merge-smoke-tests.json
+
+      - name: Upload smoke diagnostics
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: merge-smoke-diagnostics-py${{ matrix.python-version }}
+          path: ${{ runner.temp }}/smoke-diagnostics
+          if-no-files-found: ignore
+
+      - name: Remove the network namespace
+        if: always()
+        run: sudo ip netns delete smoke

.github/workflows/test-postgres.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
   workflow_dispatch:
 
 permissions:
@@ -27,6 +24,7 @@ jobs:
     timeout-minutes: ${{ matrix.job-timeout-minutes }}
     permissions:
       contents: read
+      id-token: write
 
     services:
       postgres:
@@ -97,7 +95,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip'
+        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
         timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
@@ -108,6 +106,18 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-postgres-
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
+        timeout-minutes: 5
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 12
@@ -137,9 +147,21 @@ jobs:
         env:
           TEST_PATH: ${{ matrix.test-path }}
           WORKERS: ${{ matrix.workers }}
+          PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
           else
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
           fi
+
+      - name: Upload Lens database coverage
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          use_oidc: true
+          version: v11.3.1
+          root_dir: ${{ github.workspace }}
+          files: coverage-lens-postgres.xml
+          flags: lens-postgres
+          fail_ci_if_error: true

.github/workflows/test-redis-compat.yml

@@ -4,14 +4,17 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "litellm/_redis.py"
       - "litellm/_redis_credential_provider.py"
-      - "tests/test_litellm/test_redis.py"
-      - "tests/test_litellm/caching/test_redis_connection_pool.py"
+      - "litellm/caching/redis_cache.py"
+      - "litellm/caching/evicted_client_closer.py"
+      - "tests/unit/test_redis.py"
+      - "tests/local_testing/test_caching.py"
+      - "tests/unit/caching/test_redis_connection_pool.py"
+      - "tests/unit/caching/test_redis_cluster_cache.py"
+      - "tests/unit/caching/test_evicted_client_closer.py"
       - ".github/workflows/test-redis-compat.yml"
       - "pyproject.toml"
       - "uv.lock"
@@ -28,6 +31,9 @@ jobs:
     name: "redis-py ${{ matrix.redis-version }}"
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    permissions:
+      contents: read
+      id-token: write
 
     strategy:
       fail-fast: false
@@ -57,7 +63,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router
 
       - name: Pin redis-py to the matrix version
         env:
@@ -66,12 +72,35 @@ jobs:
           uv pip install "redis==${REDIS_VERSION:?}"
           uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
 
+      - name: Build Redis for cluster authentication tests
+        run: |
+          curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz"
+          echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e  $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check
+          tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP"
+          make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server
+          echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH"
+
       - name: Run redis unit tests
         run: |
+          redis-server --version
           uv run --no-sync pytest \
-            tests/test_litellm/test_redis.py \
-            tests/test_litellm/caching/test_redis_connection_pool.py \
+            tests/unit/test_redis.py \
+            tests/unit/caching/test_redis_connection_pool.py \
+            tests/unit/caching/test_redis_cluster_cache.py \
+            tests/unit/caching/test_evicted_client_closer.py \
+            tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
+            tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
             --tb=short -vv \
             --reruns 2 \
             --reruns-delay 1 \
-            --durations=20
+            --durations=20 \
+            --cov=./litellm --cov-report=xml:coverage-redis.xml
+
+      - name: Upload Redis coverage
+        if: matrix.redis-version == '5.3.1'
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
+        with:
+          use_oidc: true
+          files: coverage-redis.xml
+          flags: redis-compat
+          fail_ci_if_error: false

.github/workflows/test-rust.yml

@@ -14,7 +14,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -24,13 +23,11 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "litellm-rust/**"
@@ -44,7 +41,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -54,7 +50,7 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
 
 permissions:
@@ -87,6 +83,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
@@ -125,6 +122,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: cargo nextest run --workspace --locked
 
@@ -166,6 +164,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: uv build --wheel --out-dir dist
 
@@ -173,7 +172,7 @@ jobs:
         env:
           RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
-      - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+      - run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
 
       - name: Run pytest tests/test_litellm_rust with the compiled extension
         run: make test-rust-extension

.github/workflows/test-semgrep.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 permissions:

.github/workflows/test-terraform-modules.yml

@@ -9,8 +9,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "terraform/litellm/aws/**"

.github/workflows/test-terraform-provider.yml

@@ -8,8 +8,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "terraform/provider/**"
@@ -79,6 +77,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -88,6 +87,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 

.github/workflows/test-unit-documentation.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -57,7 +54,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip'
+        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -67,6 +64,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/cache-cargo-build

.github/workflows/test-unit-proxy-db.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -23,6 +20,13 @@ concurrency:
 # rather than alphabetical letter ranges. Adding a new test file means adding it
 # to whichever group it belongs to, not reshuffling slices.
 #
+# `.circleci/tests.yml` runs each group's files on same-repo events under the
+# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
+# the file lists. That pipeline is manual-only while the tests migrate, so
+# `unit-flag` makes the shard run that list on every event. `test-path` keeps
+# the files that still reach real providers and never left
+# tests/proxy_unit_tests.
+#
 # Design targets:
 #   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
@@ -61,7 +65,7 @@ jobs:
   proxy-db:
     needs: assert-shard-coverage
     # Display only the semantic shard name in the checks UI instead of GHA's
-    # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
+    # default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
     # which includes every matrix field and gets truncated past the test-path.
     name: ${{ matrix.test-group }}
     permissions:
@@ -74,132 +78,93 @@ jobs:
         include:
           # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
-            test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
+            test-path: ""
+            unit-flag: proxy-db-key-generation
             workers: 0
             dist: loadscope
             timeout: 20
 
           # ---- auth: split into 2 shards ----
           - test-group: auth-checks
-            test-path: >-
-              tests/proxy_unit_tests/test_auth_checks.py
-              tests/proxy_unit_tests/test_user_api_key_auth.py
-              tests/proxy_unit_tests/test_deprecated_key_grace_period.py
+            test-path: ""
+            unit-flag: proxy-db-auth-checks
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: jwt-and-keys
-            test-path: >-
-              tests/proxy_unit_tests/test_jwt.py
-              tests/proxy_unit_tests/test_jwt_key_mapping.py
-              tests/proxy_unit_tests/test_proxy_custom_auth.py
-              tests/proxy_unit_tests/test_key_generate_dynamodb.py
+            test-path: ""
+            unit-flag: proxy-db-jwt-and-keys
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- test_proxy_utils.py, single shard, worksteal distribution ----
           - test-group: proxy-utils
-            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            test-path: ""
+            unit-flag: proxy-db-proxy-utils
             workers: 4
             dist: worksteal
             timeout: 15
 
           # ---- proxy server: split into 2 shards ----
           - test-group: proxy-server-core
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_server.py
-              tests/proxy_unit_tests/test_aproxy_startup.py
+            test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
+            unit-flag: proxy-db-proxy-server-core
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: proxy-runtime
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_config_unit_test.py
-              tests/proxy_unit_tests/test_proxy_routes.py
-              tests/proxy_unit_tests/test_server_root_path.py
-              tests/proxy_unit_tests/test_proxy_pass_user_config.py
-              tests/proxy_unit_tests/test_proxy_token_counter.py
-              tests/proxy_unit_tests/test_request_size_limit_middleware.py
-              tests/proxy_unit_tests/test_multipart_bypass_repro.py
+            test-path: ""
+            unit-flag: proxy-db-proxy-runtime
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- logging: split into 2 shards ----
           - test-group: custom-logging
-            test-path: >-
-              tests/proxy_unit_tests/test_custom_callback_input.py
-              tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
-              tests/proxy_unit_tests/test_proxy_custom_logger.py
+            test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
+            unit-flag: proxy-db-custom-logging
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: logging-misc
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_reject_logging.py
-              tests/proxy_unit_tests/test_audit_logs_proxy.py
-              tests/proxy_unit_tests/test_search_api_logging.py
+            test-path: ""
+            unit-flag: proxy-db-logging-misc
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: db-and-spend
-            test-path: >-
-              tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
-              tests/proxy_unit_tests/test_db_schema_changes.py
-              tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
-              tests/proxy_unit_tests/test_skills_db.py
-              tests/proxy_unit_tests/test_update_daily_tag_spend.py
-              tests/proxy_unit_tests/test_update_spend.py
-              tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
+            test-path: ""
+            unit-flag: proxy-db-db-and-spend
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- guardrails + budget + hooks: split into 2 ----
           - test-group: guardrails-hooks
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_setting_guardrails.py
-              tests/proxy_unit_tests/test_banned_keyword_list.py
-              tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
+            test-path: ""
+            unit-flag: proxy-db-guardrails-hooks
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: budgets
-            test-path: >-
-              tests/proxy_unit_tests/test_default_end_user_budget_simple.py
-              tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
-              tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
+            test-path: ""
+            unit-flag: proxy-db-budgets
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: endpoints-and-responses
-            test-path: >-
-              tests/proxy_unit_tests/test_blog_posts_endpoint.py
-              tests/proxy_unit_tests/test_models_fallback_endpoint.py
-              tests/proxy_unit_tests/test_google_endpoint_routing.py
-              tests/proxy_unit_tests/test_google_gemini_proxy_request.py
-              tests/proxy_unit_tests/test_gemini_agents_endpoints.py
-              tests/proxy_unit_tests/test_get_favicon.py
-              tests/proxy_unit_tests/test_get_image.py
-              tests/proxy_unit_tests/test_reducto_ocr_route.py
-              tests/proxy_unit_tests/test_ui_path_detection.py
-              tests/proxy_unit_tests/test_prompt_test_endpoint.py
-              tests/proxy_unit_tests/test_check_batch_cost.py
-              tests/proxy_unit_tests/test_check_responses_cost.py
-              tests/proxy_unit_tests/test_response_polling_handler.py
-              tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
-              tests/proxy_unit_tests/test_realtime_cache.py
-              tests/proxy_unit_tests/test_proxy_exception_mapping.py
-              tests/proxy_unit_tests/test_custom_tokenizer_bug.py
+            test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
+            unit-flag: proxy-db-endpoints-and-responses
             workers: 4
             dist: loadscope
             timeout: 15
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}
+      unit-flag: ${{ matrix.unit-flag }}
       workers: ${{ matrix.workers }}
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}

.github/workflows/test-unit.yml

@@ -4,13 +4,10 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
   push:
     branches:
       - main
-      - litellm_internal_staging
   workflow_dispatch:
 
 permissions:
@@ -34,10 +31,14 @@ concurrency:
 # number, so a partially-specified entry would fail the call rather than fall
 # back to the default.
 #
-# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
-# already a matrix and carries a shard-coverage guard that reads that file by
-# name. Folding it in here is a follow-up, together with generalising that guard
-# into assert_ci_coverage.py.
+# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already
+# a matrix and carries a shard-coverage guard that reads that file by name.
+# Folding it in here is a follow-up, together with generalising that guard into
+# assert_ci_coverage.py.
+#
+# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the
+# shard under the same Codecov flag. That pipeline is manual-only while the
+# tests migrate, so the shard also runs those files on every event.
 jobs:
   unit:
     name: ${{ matrix.shard }}
@@ -51,7 +52,8 @@ jobs:
         include:
           - shard: mcp-integration
             artifact-name: mcp-integration
-            test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client"
+            test-path: "tests/mcp_tests"
+            unit-flag: mcp-integration
             workers: 2
             reruns: 0
             timeout-minutes: 20
@@ -59,7 +61,8 @@ jobs:
 
           - shard: core-utils
             artifact-name: core-utils
-            test-path: "tests/test_litellm/litellm_core_utils"
+            test-path: ""
+            unit-flag: core-utils
             workers: 2
             reruns: 1
             timeout-minutes: 20
@@ -67,11 +70,8 @@ jobs:
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
-            test-path: >-
-              tests/test_litellm/enterprise
-              tests/test_litellm/google_genai
-              tests/test_litellm/router_utils
-              tests/test_litellm/router_strategy
+            test-path: ""
+            unit-flag: enterprise-routing
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -79,7 +79,10 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: "tests/test_litellm/integrations"
+            test-path: >-
+              tests/test_litellm/integrations
+              tests/test_litellm/tracing
+            unit-flag: integrations
             workers: 2
             reruns: 3
             timeout-minutes: 20
@@ -87,7 +90,8 @@ jobs:
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
-            test-path: "tests/test_litellm/llms/vertex_ai"
+            test-path: ""
+            unit-flag: llm-vertex-ai
             workers: 1
             reruns: 2
             timeout-minutes: 20
@@ -95,7 +99,8 @@ jobs:
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
-            test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            test-path: ""
+            unit-flag: llm-other-providers
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -104,24 +109,8 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/test_litellm/batches
-              tests/test_litellm/secret_managers
-              tests/test_litellm/a2a_protocol
-              tests/test_litellm/chat_completions
-              tests/test_litellm/completion_extras
-              tests/test_litellm/containers
-              tests/test_litellm/endpoints
-              tests/test_litellm/images
-              tests/test_litellm/interactions
-              tests/test_litellm/messages
-              tests/test_litellm/ocr
-              tests/test_litellm/passthrough
-              tests/test_litellm/rag
-              tests/test_litellm/rerank_api
-              tests/test_litellm/rust_bridge
-              tests/test_litellm/vector_stores
-              tests/test_litellm/videos
               tests/test_litellm/test_*.py
+            unit-flag: misc
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -130,10 +119,19 @@ jobs:
           - shard: proxy-auth
             artifact-name: proxy-auth
             test-path: >-
-              tests/test_litellm/proxy/auth
-              tests/test_litellm/proxy/hooks
-              tests/test_litellm/proxy/policy_engine
-              tests/test_litellm/proxy/client
+              tests/unit/proxy/auth
+              tests/unit/proxy/hooks
+              tests/unit/proxy/policy_engine
+              tests/unit/proxy/client
+              --ignore=tests/unit/proxy/auth/test_auth_checks.py
+              --ignore=tests/unit/proxy/auth/test_user_api_key_auth.py
+              --ignore=tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+              --ignore=tests/unit/proxy/auth/test_jwt.py
+              --ignore=tests/unit/proxy/auth/test_models_fallback_endpoint.py
+              --ignore=tests/unit/proxy/auth/test_multipart_bypass_repro.py
+              --ignore=tests/unit/proxy/auth/test_proxy_routes.py
+              --ignore=tests/unit/proxy/hooks/test_banned_keyword_list.py
+              --ignore=tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -142,38 +140,46 @@ jobs:
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
             test-path: >-
-              tests/test_litellm/proxy/analytics_endpoints
-              tests/test_litellm/proxy/management_endpoints
-              tests/test_litellm/proxy/list_api
-              tests/test_litellm/proxy/memory
-              tests/test_litellm/proxy/guardrails
-              tests/test_litellm/proxy/management_helpers
-              tests/test_litellm/proxy/anthropic_endpoints
-              tests/test_litellm/proxy/google_endpoints
-              tests/test_litellm/proxy/openai_files_endpoint
-              tests/test_litellm/proxy/batches_endpoints
-              tests/test_litellm/proxy/container_endpoints
-              tests/test_litellm/proxy/fine_tuning_endpoints
-              tests/test_litellm/proxy/vector_store_files_endpoints
-              tests/test_litellm/proxy/video_endpoints
-              tests/test_litellm/proxy/response_api_endpoints
-              tests/test_litellm/proxy/image_endpoints
-              tests/test_litellm/proxy/ocr_endpoints
-              tests/test_litellm/proxy/vector_store_endpoints
-              tests/test_litellm/proxy/agent_endpoints
-              tests/test_litellm/proxy/a2a
-              tests/test_litellm/proxy/credential_endpoints
-              tests/test_litellm/proxy/discovery_endpoints
-              tests/test_litellm/proxy/health_endpoints
-              tests/test_litellm/proxy/shutdown
-              tests/test_litellm/proxy/public_endpoints
-              tests/test_litellm/proxy/prompts
-              tests/test_litellm/proxy/rag_endpoints
-              tests/test_litellm/proxy/rerank_endpoints
-              tests/test_litellm/proxy/realtime_endpoints
-              tests/test_litellm/proxy/ui_crud_endpoints
-              tests/test_litellm/proxy/config_resolvers
-              tests/test_litellm/proxy/utils
+              tests/unit/proxy/analytics_endpoints
+              tests/unit/proxy/management_endpoints
+              tests/unit/proxy/list_api
+              tests/unit/proxy/memory
+              tests/unit/proxy/guardrails
+              tests/unit/proxy/management_helpers
+              --ignore=tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
+              --ignore=tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
+              --ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+              --ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
+              --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
+              --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
+              --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
+              --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
+              tests/unit/proxy/anthropic_endpoints
+              tests/unit/proxy/google_endpoints
+              tests/unit/proxy/openai_files_endpoint
+              tests/unit/proxy/batches_endpoints
+              tests/unit/proxy/container_endpoints
+              tests/unit/proxy/fine_tuning_endpoints
+              tests/unit/proxy/vector_store_files_endpoints
+              tests/unit/proxy/video_endpoints
+              tests/unit/proxy/response_api_endpoints
+              tests/unit/proxy/image_endpoints
+              tests/unit/proxy/ocr_endpoints
+              tests/unit/proxy/vector_store_endpoints
+              tests/unit/proxy/agent_endpoints
+              tests/unit/proxy/a2a
+              tests/unit/proxy/credential_endpoints
+              tests/unit/proxy/discovery_endpoints
+              tests/unit/proxy/health_endpoints
+              tests/unit/proxy/shutdown
+              tests/unit/proxy/public_endpoints
+              tests/unit/proxy/prompts
+              tests/unit/proxy/rag_endpoints
+              tests/unit/proxy/rerank_endpoints
+              tests/unit/proxy/realtime_endpoints
+              tests/unit/proxy/ui_crud_endpoints
+              tests/unit/proxy/config_resolvers
+              tests/unit/proxy/utils
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -181,7 +187,7 @@ jobs:
 
           - shard: proxy-server
             artifact-name: proxy-server
-            test-path: "tests/test_litellm/proxy/proxy_server"
+            test-path: "tests/unit/proxy/proxy_server"
             workers: 4
             reruns: 2
             timeout-minutes: 60
@@ -190,18 +196,61 @@ jobs:
           - shard: proxy-infra
             artifact-name: proxy-infra
             test-path: >-
-              tests/test_litellm/proxy/db
-              tests/test_litellm/proxy/middleware
-              tests/test_litellm/proxy/spend_tracking
-              tests/test_litellm/proxy/pass_through_endpoints
-              tests/test_litellm/proxy/_experimental
-              tests/test_litellm/proxy/experimental
-              tests/test_litellm/proxy/common_utils
-              tests/test_litellm/proxy/enterprise_billing
-              tests/test_litellm/proxy/types_utils
-              tests/test_litellm/proxy/logging_endpoints
-              tests/test_litellm/proxy/test_*.py
-              tests/test_gateway
+              tests/unit/proxy/db
+              --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
+              --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py
+              tests/unit/proxy/middleware
+              --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py
+              tests/unit/proxy/spend_tracking
+              --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py
+              tests/unit/proxy/pass_through_endpoints
+              tests/unit/proxy/_experimental
+              --ignore=tests/unit/proxy/_experimental/mcp_server
+              tests/unit/proxy/experimental
+              tests/unit/proxy/common_utils
+              --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py
+              --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py
+              --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py
+              --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
+              --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py
+              tests/unit/proxy/enterprise_billing
+              tests/unit/proxy/types_utils
+              tests/unit/proxy/logging_endpoints
+            unit-flag: proxy-infra
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: proxy-infra-root
+            artifact-name: proxy-infra-root
+            test-path: >-
+              tests/unit/proxy/test_*.py
+              --ignore=tests/unit/proxy/test_aproxy_startup.py
+              --ignore=tests/unit/proxy/test_credential_slot_registry.py
+              --ignore=tests/unit/proxy/test_custom_callback_input.py
+              --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py
+              --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py
+              --ignore=tests/unit/proxy/test_db_schema_changes.py
+              --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py
+              --ignore=tests/unit/proxy/test_get_favicon.py
+              --ignore=tests/unit/proxy/test_get_image.py
+              --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py
+              --ignore=tests/unit/proxy/test_prompt_test_endpoint.py
+              --ignore=tests/unit/proxy/test_proxy_config_unit_test.py
+              --ignore=tests/unit/proxy/test_proxy_custom_auth.py
+              --ignore=tests/unit/proxy/test_proxy_reject_logging.py
+              --ignore=tests/unit/proxy/test_proxy_server.py
+              --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py
+              --ignore=tests/unit/proxy/test_proxy_token_counter.py
+              --ignore=tests/unit/proxy/test_proxy_utils.py
+              --ignore=tests/unit/proxy/test_reducto_ocr_route.py
+              --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py
+              --ignore=tests/unit/proxy/test_server_root_path.py
+              --ignore=tests/unit/proxy/test_ui_path_detection.py
+              --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py
+              --ignore=tests/unit/proxy/test_update_spend.py
+              --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -209,11 +258,8 @@ jobs:
 
           - shard: caching-local
             artifact-name: caching-local
-            test-path: >-
-              tests/local_testing/test_cache_preset_key.py
-              tests/local_testing/test_caching_handler.py
-              tests/local_testing/test_responses_stream_cache_keys.py
-              tests/local_testing/test_unit_test_caching.py
+            test-path: ""
+            unit-flag: caching-local
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -221,7 +267,8 @@ jobs:
 
           - shard: proxy-extras
             artifact-name: proxy-extras
-            test-path: "tests/litellm-proxy-extras"
+            test-path: ""
+            unit-flag: proxy-extras
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -229,7 +276,8 @@ jobs:
 
           - shard: enterprise-package
             artifact-name: enterprise-package
-            test-path: "tests/enterprise"
+            test-path: ""
+            unit-flag: enterprise-package
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -237,10 +285,8 @@ jobs:
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
-            test-path: >-
-              tests/test_litellm/responses
-              tests/test_litellm/caching
-              tests/test_litellm/types
+            test-path: ""
+            unit-flag: responses-caching-types
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -248,6 +294,7 @@ jobs:
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}
+      unit-flag: ${{ matrix.unit-flag || '' }}
       workers: ${{ matrix.workers }}
       reruns: ${{ matrix.reruns }}
       timeout-minutes: ${{ matrix.timeout-minutes }}

.github/workflows/test-vscode-extension.yml

@@ -6,8 +6,6 @@ on:
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
     paths:
       - "vscode-extension/**"

.github/workflows/zizmor.yml

@@ -2,12 +2,10 @@ name: GitHub Actions Security Analysis
 
 on:
   push:
-    branches: [main, litellm_internal_staging]
+    branches: [main]
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
-      - litellm_oss_staging
       - "litellm_**"
 
 concurrency:

.gitignore

@@ -104,7 +104,7 @@ litellm_config.yaml
 .cursor
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
-tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
 scripts/test_vertex_ai_search.py
 LAZY_LOADING_IMPROVEMENTS.md
 STABILIZATION_TODO.md

AGENTS.md

@@ -27,17 +27,17 @@ Never test structure of code only function of it
 
 A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
 
-`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
+`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
 
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
-When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
 
 Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
 
-If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
+If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
 
@@ -62,7 +62,7 @@ Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `
 
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
-If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
+If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
 
@@ -96,6 +96,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
   - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
   - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
+  - Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

ARCHITECTURE.md

@@ -255,7 +255,7 @@ Conventions to follow when touching this layer:
 | Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. |
 | Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. |
 
-To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`.
+To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/unit/repositories/`.
 
 ---
 
@@ -336,7 +336,7 @@ Each translation is isolated in its own file, making it easy to test and modify
 | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
 | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
 | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
-| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
 | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
 | `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
 | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |

CONTRIBUTING.md

@@ -14,7 +14,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
+  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
 
 #### UI PRs
 
@@ -72,7 +72,7 @@ make format
 make lint
 
 # Run the tests covering your change (CI runs the full suite)
-uv run pytest tests/test_litellm/<your_test_file>.py -v
+uv run pytest tests/unit/<your_test_file>.py -v
 
 # Commit your changes (must follow Conventional Commits — see above)
 git add .
@@ -88,7 +88,7 @@ git push origin feature/your-feature
 
 ### Where to Add Tests
 
-Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+Add your tests to the [`tests/unit/` directory](https://github.com/BerriAI/litellm/tree/main/tests/unit).
 
 - This directory mirrors the structure of the `litellm/` directory
 - **Only add mocked tests** - no real LLM API calls in this directory
@@ -96,10 +96,10 @@ Add your tests to the [`tests/test_litellm/` directory](https://github.com/Berri
 
 ### File Naming Convention
 
-The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+The `tests/unit/` directory follows the same structure as `litellm/`:
 
-- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
-- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+- `litellm/proxy/caching_routes.py` → `tests/unit/proxy/test_caching_routes.py`
+- `litellm/utils.py` → `tests/unit/test_utils.py`
 
 ### Example Test
 
@@ -125,10 +125,10 @@ def test_your_feature():
 
 Run the tests covering your change:
 ```bash
-uv run pytest tests/test_litellm/test_your_file.py -v
+uv run pytest tests/unit/test_your_file.py -v
 ```
 
-`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
+`tests/unit` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
 
 If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
 
@@ -136,7 +136,7 @@ If you're running broader test suites, proxy tests, or anything that touches Pos
 make install-test-deps
 ```
 
-This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs.
+This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary`, `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs.
 
 ### Running Linting and Formatting Checks
 

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

Makefile

@@ -1,10 +1,10 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
+.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	test-rust-extension \
+	test-rust-extension rust-sqlx-prepare \
 	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
@@ -42,20 +42,22 @@ help:
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit          - Run unit tests (tests/unit and tests/test_litellm)"
 	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
 	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
 	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
 	@echo "  make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
+	@echo "  make test-unit-proxy-root - Run proxy root-file tests (tests/unit/proxy/test_*.py)"
 	@echo "  make test-unit-integrations - Run integration tests (~60 files)"
 	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
 	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
 	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
-	@echo "  make test-proxy-unit-a  - Run proxy_unit_tests (a-o, ~20 files)"
-	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
+	@echo "  make test-proxy-unit-a  - Run tests/unit/proxy (a-o)"
+	@echo "  make test-proxy-unit-b  - Run tests/unit/proxy (p-z)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
+	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
 	@echo ""
 	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
 	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
@@ -301,48 +303,54 @@ test-rust-extension:
 	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
 	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
 	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
-		--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
+		--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
 		litellm.rust_bridge._native && \
 	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
 	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
 
+rust-sqlx-prepare:
+	cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
+
 test: install-test-deps
 	$(UV_RUN) pytest tests/
 
 test-unit: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
+	$(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4
 
 # Matrix test targets (matching CI workflow groups)
 test-unit-llms: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/llms --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-guardrails: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/guardrails tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-core: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-misc: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-root: install-test-deps
+	$(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20
 
 test-unit-core-utils: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20
 
 test-unit-other: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
 
-# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+# Proxy unit tests (tests/unit/proxy split alphabetically)
 test-proxy-unit-a: install-test-deps
-	$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy --ignore-glob='tests/unit/proxy/test_[p-z]*.py' --tb=short -vv -n 2 --durations=20
 
 test-proxy-unit-b: install-test-deps
-	$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py tests/unit/skills --tb=short -vv -n 2 --durations=20
 
 test-integration: install-test-deps
 	$(UV_RUN) pytest tests/ -k "not test_litellm"

README.md

@@ -268,6 +268,31 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 
 </details>
 
+<details>
+<summary><b>Agents</b> - Run Claude Code, Codex, OpenCode or Deep Agents on any model (Python SDK)</summary>
+
+### Python SDK - Agents
+
+```python
+import litellm
+from litellm import Harness, sandbox
+
+result = litellm.agent(
+    Harness.CLAUDE_CODE,  # or Harness.CODEX, Harness.OPENCODE, Harness.DEEPAGENTS
+    "Find why tests/test_router.py is flaky and fix it.",
+    sandbox=sandbox.local("./repo"),
+    model="litellm_proxy/claude-sonnet-4-5",  # a model group on your AI Gateway
+)
+
+print(result.text, result.cost, [f.path for f in result.files])
+```
+
+Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call the agent makes goes through your AI Gateway, tagged `harness,claude_code`. Drop the `litellm_proxy/` prefix to call a provider directly. Install `starlette uvicorn` plus the agent's CLI (`claude`, `codex` or `opencode`), or `deepagents langchain-litellm` for Deep Agents.
+
+[**Docs: Agent Harnesses**](https://docs.litellm.ai/docs/harness)
+
+</details>
+
 ### Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
 
 | Provider                                                                            | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
@@ -362,6 +387,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

backend/routes/allowlist.py

@@ -26,6 +26,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/v2/login",
     "/v3/login",
     "/logout",
+    "/session/logout",
     "/token",
     "/onboarding/",
     "/audit",
@@ -59,6 +60,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Tools / agents (registry & policy admin)
     "/v1/tool/",
     "/v1/agents",
+    "/agent/daily/activity/",
     # Guardrails admin
     "/v2/guardrails/",
     # MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
@@ -80,6 +82,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Spend / analytics
     "/spend/",
     "/analytics/",
+    "/lens/",
+    "/v1/traces",
     "/global/",
     "/user_agent",
     "/usage/",
@@ -143,6 +147,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
     {
         "/",
         "/routes",
+        "/lens",
         "/openapi.json",
         "/docs",
         "/docs/oauth2-redirect",

basedpyright-code-budget.json

@@ -99,7 +99,7 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44358
+    "limit": 44802
   },
   "reportUnknownLambdaType": {
     "limit": 109

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json

@@ -1697,6 +1697,63 @@
       "title": "litellm_video_duration_seconds_metric rate",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "Share of the provider's bill LiteLLM captured as spend over the scheduled capture-rate check's window (needs general_settings.spend_capture_rate_check); NaN while no rate is available",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 107
+      },
+      "id": 111,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max by (api_provider) (litellm_spend_capture_rate)",
+          "legendFormat": "{{api_provider}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "litellm_spend_capture_rate",
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
       "gridPos": {

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md

@@ -1,8 +1,8 @@
 # LiteLLM All Prometheus Metrics dashboard
 
-Every `litellm_*` metric family the proxy can expose on `/metrics` (134 families across 95 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
+Every `litellm_*` metric family the proxy can expose on `/metrics` (136 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
 
-Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
 
 The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
 

cookbook/misc/config.yaml

@@ -24,7 +24,7 @@ model_list:
   - model_name: sagemaker-completion-model
     litellm_params:
       model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4
-      input_cost_per_second: 0.000420  
+      cost_per_second: 0.000420
   - model_name: text-embedding-ada-002
     litellm_params: 
       model: azure/azure-embedding-model

cookbook/misc/test_responses_api.py

@@ -12,7 +12,7 @@ def encode_image(image_path):
 
 
 # Path to your image
-image_path = "litellm/proxy/logo.jpg"
+image_path = "litellm/proxy/logo.png"
 
 # Getting the Base64 string
 base64_image = encode_image(image_path)
@@ -27,7 +27,7 @@ response = client.responses.create(
                 {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
-                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                    "image_url": f"data:image/png;base64,{base64_image}",
                 },
             ],
         }

db_scripts/backfill_key_total_spend.sql

@@ -0,0 +1,43 @@
+-- One-shot backfill of LiteLLM_VerificationToken.total_spend (lifetime spend)
+-- for keys created before the column was introduced in LiteLLM v1.103.0.
+--
+-- The column was added with DEFAULT 0 and no backfill, so keys that predate
+-- the upgrade report lifetime spend below their current period spend. New
+-- deployments do not need this script: total_spend is updated at request
+-- time from the moment the release is deployed. Run it only if you want
+-- pre-upgrade keys to show their historical lifetime spend. It sets lifetime
+-- spend to at least the current spend on every key, active and archived,
+-- because current period spend is a valid lower bound on lifetime spend.
+-- For keys with no budget reset that is already the exact lifetime value;
+-- for resetting keys it only recovers the current period. It is idempotent:
+-- it only touches rows where total_spend is below spend, so re-running is a
+-- no-op. It touches no spend logs and runs in seconds.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. A key "resets" when its own budget_duration IS NOT NULL, or when its
+--    budget_id links to a LiteLLM_BudgetTable row whose budget_duration IS
+--    NOT NULL (a linked budget resets the key's spend each period too). For
+--    those keys this script only recovers the current period;
+--    db_scripts/backfill_key_total_spend_from_spend_logs.sql is an optional
+--    follow-up that rebuilds the earlier periods from LiteLLM_SpendLogs.
+--
+-- 3. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend.sql
+
+UPDATE "LiteLLM_VerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+UPDATE "LiteLLM_DeletedVerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

db_scripts/backfill_key_total_spend_from_spend_logs.sql

@@ -0,0 +1,89 @@
+-- Optional follow-up to db_scripts/backfill_key_total_spend.sql. Run that
+-- script first; this one rebuilds earlier budget periods for the keys it
+-- can only partially fix: keys whose spend resets each period, because their own
+-- budget_duration IS NOT NULL or because their budget_id links to a
+-- LiteLLM_BudgetTable row whose budget_duration IS NOT NULL.
+--
+-- For those keys the "spend" column only covers the current period, so
+-- lifetime spend is reconstructed from LiteLLM_SpendLogs. The join matches
+-- l.api_key against both the stored token and its second sha256
+-- (encode(sha256(convert_to(token, 'UTF8')), 'hex')), because spend logs
+-- written by older paths recorded the re-hashed digest instead of the
+-- token. It is idempotent and never lowers a value: every statement only
+-- touches rows where total_spend is below the rebuilt sum, so re-running is
+-- a no-op, and a key whose log history is shorter than its current period
+-- keeps the value backfill_key_total_spend.sql already gave it.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. It requires spend logs to have been enabled, and coverage is bounded
+--    by maximum_spend_logs_retention_period: spend older than the retention
+--    window is already gone and cannot be recovered.
+--
+-- 3. On a large SpendLogs table the join scan is slow, so run it off peak.
+--
+-- 4. Run it while the proxy is idle (or with traffic paused). The proxy
+--    flushes spend logs in batches, so a request that already raised
+--    total_spend but whose log is still queued is missing from the sum, and
+--    the rebuilt value would be short by that in-flight amount.
+--
+-- 5. A custom token can be deleted and recreated, so the archived table can
+--    hold several lifetimes of one token. The update only rewrites archived
+--    rows that reset, and the log sum covers every lifetime of that token.
+--
+-- 6. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend_from_spend_logs.sql
+
+-- Active keys whose spend resets (own budget_duration, or a linked
+-- LiteLLM_BudgetTable row with one). Rebuild from LiteLLM_SpendLogs,
+-- matching api_key against the stored token and its second sha256 digest.
+UPDATE "LiteLLM_VerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM "LiteLLM_VerificationToken" k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    WHERE k2.budget_duration IS NOT NULL
+       OR k2.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       )
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend;
+
+-- Archived tokens are not unique, so collapse them to one row per token
+-- before joining spend logs; the update then hits every resetting archived
+-- row.
+UPDATE "LiteLLM_DeletedVerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM (
+        SELECT DISTINCT token
+        FROM "LiteLLM_DeletedVerificationToken"
+        WHERE budget_duration IS NOT NULL
+           OR budget_id IN (
+               SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+           )
+    ) k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend
+  AND (k.budget_duration IS NOT NULL
+       OR k.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       ));
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

deploy/lens/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
+COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
+USER 65532:65532
+CMD ["python", "-m", "lens.worker"]

deploy/lens/Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+**
+!litellm/
+!litellm/proxy/
+!litellm/proxy/lens/
+!litellm/proxy/lens/__init__.py
+!litellm/proxy/lens/models.py
+!litellm/proxy/lens/analysis.py
+!litellm/proxy/lens/worker.py

deploy/lens/README.md

@@ -0,0 +1,117 @@
+# Lens worker
+
+Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
+
+## Start a worker
+
+Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
+
+In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in
+
+The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
+
+The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds
+
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+
+```bash
+docker compose --env-file /path/to/lens.env -f compose.yaml up -d
+```
+
+Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
+
+The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
+
+The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
+
+If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
+
+V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
+
+## Configure a lens
+
+Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute
+
+Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage
+
+Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
+
+Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
+
+## Read the results
+
+Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
+
+Use the batch selector or Scans tab to reopen previous results. Each batch keeps its own findings, settings, selected runs, coverage and cost. Older batches created before snapshot support remain available through accumulated findings. The Runs tab lists the selected batch's sample and can filter per-run observations, including runs without an observed issue and runs with insufficient evidence. These observations precede the final evidence investigation. Linked-run counts on findings include cited counterexamples, so they are not failure counts
+
+Choose **This is expected** and explain why to teach later scans about acceptable behavior. Feedback is kept with the lens and included in subsequent reviews. It does not alter historical evidence or exempt different problems
+
+## What a scan does
+
+The proxy selects executions received or updated within the configured lookback window, with a two-minute settling period. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID
+
+A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
+
+The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence
+
+There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools
+
+Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
+
+Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
+
+## Operations and limits
+
+PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
+
+Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Both the Lens budget and the selected virtual key’s budgets, model permissions, and rate limits apply. Analysis spend appears under that key in Virtual Keys and normal request logs, with Lens, scan, and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Existing workers need a billing key assigned in **Set up analysis** before they can resume
+
+V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them
+
+
+## API access
+
+The UI and API use the same scan lifecycle. Authenticate with a proxy administrator credential for writes, or a proxy-admin viewer credential for reads. Worker credentials are only for worker operations
+
+```bash
+curl "$LITELLM_URL/lens" -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H 'Content-Type: application/json' -d '{
+    "name": "Research quality", "model": "your-model-alias",
+    "context": "Answer the requested question using cited, retrieved evidence.",
+    "source": "traces", "lookback_hours": 24,
+    "sample_percent": 100, "sample_size": null, "concurrency": 8,
+    "enabled": true, "interval_minutes": 1440, "monthly_budget": 50
+  }'
+
+curl "$LITELLM_URL/lens/$LENS_ID/runs" -X POST \
+  -H "Authorization: Bearer $LITELLM_API_KEY" -H 'Content-Type: application/json' -d '{}'
+
+curl "$LITELLM_URL/lens/$LENS_ID/runs?offset=0" -H "Authorization: Bearer $LITELLM_API_KEY"
+curl "$LITELLM_URL/lens/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITELLM_API_KEY"
+```
+
+Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/lens/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /lens/{id}/findings/{finding_id}` with `status` and `reason`
+
+## Quality evaluation
+
+Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
+
+```bash
+python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
+  --model your-model-alias --split all --background 1000 --concurrency 16 \
+  --output /tmp/lens-quality.json
+```
+
+Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
+
+The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
+
+To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
+
+## Upgrading from the original Lens API
+
+The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`, and worker claims use `lens_id`. Upgrade the proxy and recreate every worker with the image shown by the upgraded dashboard before starting new scans. Update API clients to the new paths and response fields. Old worker images cannot poll the renamed API
+
+Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
+
+Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push