fix(proxy): resolve mainline drift in end-user budget enforcement

Preserve the relocated shared fallback helper and upstream script removal.
Retain opt-in master-key budget enforcement and verified JWT customer handling.

63 focused auth/budget/fallback regression cases pass on current main.

.circleci/config.yml

@@ -38,7 +38,7 @@ commands:
     parameters:
       category:
         type: enum
-        enum: ["backend", "client", "provider-harness"]
+        enum: ["backend", "client", "provider-harness", "redis-compat"]
         default: "backend"
     steps:
       - run:
@@ -229,6 +229,19 @@ commands:
       - wait_for_service:
           url: tcp://localhost:6379
           timeout: "60"
+  install_codecov_cli:
+    steps:
+      - run:
+          name: Install Codecov CLI (pinned v11.3.1)
+          when: always
+          command: |
+            curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
+            curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
+            [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d  codecov" ]
+            (cd /tmp && sha256sum -c codecov.SHA256SUM)
+            chmod +x /tmp/codecov
+            mkdir -p "$HOME/.local/bin"
+            mv /tmp/codecov "$HOME/.local/bin/codecov"
   start_openai_record_replay_proxy:
     description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
     steps:
@@ -1693,6 +1706,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -1716,9 +1734,8 @@ jobs:
           command: |
             docker run -d \
               -p 4001:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               --name schema-seed \
               --add-host=host.docker.internal:host-gateway \
               -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
@@ -1789,6 +1806,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -1812,9 +1834,8 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e USE_PRISMA_MIGRATE=True \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e AZURE_API_KEY=$AZURE_API_KEY \
@@ -1859,7 +1880,7 @@ jobs:
           name: Seed the routing strategy through /config/update
           command: |
             curl --noproxy '*' -sSf -X POST http://localhost:4000/config/update \
-              -H 'Authorization: Bearer sk-1234' -H 'Content-Type: application/json' \
+              -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
               -d '{"router_settings": {"routing_strategy": "usage-based-routing-v2"}}'
       - run:
           name: Run tests
@@ -1888,6 +1909,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -1909,9 +1935,8 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e AZURE_API_KEY=$AZURE_API_KEY \
               -e AZURE_API_BASE=$AZURE_API_BASE \
               -e AZURE_API_VERSION="2024-05-01-preview" \
@@ -1974,6 +1999,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -1999,12 +2029,11 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
@@ -2060,12 +2089,11 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE="bad-license" \
@@ -2109,6 +2137,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -2138,11 +2171,10 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=host.docker.internal \
               -e REDIS_PORT=6379 \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
@@ -2197,6 +2229,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -2221,12 +2258,11 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e USE_DDTRACE=True \
@@ -2244,12 +2280,11 @@ jobs:
           command: |
             docker run -d \
               -p 4001:4001 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e USE_DDTRACE=True \
@@ -2296,6 +2331,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -2322,10 +2362,9 @@ jobs:
             docker run -d \
               --restart on-failure \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e STORE_MODEL_IN_DB="True" \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
@@ -2375,6 +2414,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       # Remove Docker CLI installation since it's already available in machine executor
@@ -2397,12 +2441,11 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
@@ -2459,6 +2502,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -2480,9 +2528,8 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e GEMINI_API_KEY=$GEMINI_API_KEY \
               -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
@@ -2550,6 +2597,11 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
@@ -2572,9 +2624,8 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
               -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
               -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
@@ -2600,7 +2651,7 @@ jobs:
           command: |
             mkdir -p test-results
             export LITELLM_PROXY_URL="http://localhost:4000"
-            export LITELLM_API_KEY="sk-1234"
+            export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
             TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
@@ -2663,6 +2714,11 @@ jobs:
       PROXY_LOGOUT_URL: "https://www.example.com"
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
@@ -2753,8 +2809,6 @@ jobs:
       - run:
           name: Start LiteLLM proxy
           environment:
-            LITELLM_MASTER_KEY: "sk-1234"
-            LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
             MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
             DISABLE_SCHEMA_UPDATE: "true"
             SERVER_ROOT_PATH: ""
@@ -2774,7 +2828,7 @@ jobs:
           name: Wait for proxy to be ready
           command: |
             for i in $(seq 1 60); do
-              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
               if [ "$HTTP_CODE" = "200" ]; then
                 echo "Proxy is ready"
                 exit 0
@@ -2821,6 +2875,11 @@ jobs:
       SERVER_ROOT_PATH: "/litellm"
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
@@ -2897,8 +2956,6 @@ jobs:
       - run:
           name: Start LiteLLM proxy under a server root path
           environment:
-            LITELLM_MASTER_KEY: "sk-1234"
-            LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
             MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
             DISABLE_SCHEMA_UPDATE: "true"
           # Output flows to this step's own log, so a boot crash is visible here
@@ -2913,7 +2970,7 @@ jobs:
           name: Wait for prefixed proxy to be ready
           command: |
             for i in $(seq 1 60); do
-              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
               if [ "$HTTP_CODE" = "200" ]; then
                 echo "Prefixed proxy is ready"
                 exit 0
@@ -3279,30 +3336,164 @@ jobs:
       - store_artifacts:
           path: test-results
 
-  unit:
+  postgres_suite:
+    parameters:
+      test_path:
+        type: string
+      seed:
+        type: boolean
+      coverage_flag:
+        type: string
+        default: ""
+      timeout_minutes:
+        type: integer
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
-    parallelism: 4
+    environment:
+      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
     steps:
+      - checkout
+      - skip_if_unrelated_changes
+      - setup_litellm_test_deps
+      - start_postgres:
+          db_name: litellm_test
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+      - run:
+          name: Generate Prisma client
+          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+      - when:
+          condition: << parameters.seed >>
+          steps:
+            - run:
+                name: Seed database schema
+                command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+      - run:
+          name: Run << parameters.test_path >>
+          command: |
+            mkdir -p test-results
+            coverage_args=()
+            if [ -n "<< parameters.coverage_flag >>" ]; then
+              coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml)
+            fi
+            timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \
+              << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \
+              --junitxml=test-results/junit.xml "${coverage_args[@]}"
+      - when:
+          condition: << parameters.coverage_flag >>
+          steps:
+            - install_codecov_cli
+            - run:
+                name: Upload coverage
+                when: always
+                command: |
+                  [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
+                  codecov upload-process --disable-search --fail-on-error -f coverage.xml \
+                    -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \
+                    -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
+  mcp_integration:
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      COVERAGE_CORE: sysmon
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+    steps:
+      - checkout
+      - skip_if_unrelated_changes
       - setup_litellm_test_deps
       - run:
           name: Generate Prisma client
           command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
       - run:
-          name: Run unit tests
+          name: Install MCP SDK1 peer
           command: |
-            mkdir -p test-results/unit
-            shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
-            if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
-            mapfile -t files < <(printf '%s\n' "${shard}")
-            set +e
-            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
-            status=$?
-            set -e
-            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
-            exit "$status"
+            uv venv --python 3.12 .venv-mcp-peer
+            uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+            echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
+      - run:
+          name: Run MCP integration tests
+          command: |
+            mkdir -p test-results
+            env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
+              timeout --signal=TERM 20m uv run --no-sync pytest \
+              tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
+              tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
+              --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
+              --cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
+      - install_codecov_cli
+      - run:
+          name: Upload MCP integration coverage
+          when: always
+          command: |
+            [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
+            codecov upload-process --disable-search --fail-on-error -f coverage.xml \
+              -F mcp-integration -C "$CIRCLE_SHA1" \
+              -n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
+  redis_compat:
+    parameters:
+      redis_py:
+        type: string
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - checkout
+      - skip_if_unrelated_changes:
+          category: redis-compat
+      - setup_litellm_test_deps
+      - run:
+          name: Pin redis-py version
+          command: |
+            uv pip install "redis==<< parameters.redis_py >>"
+            uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
+      - run:
+          name: Install redis-server 7.2.16
+          command: |
+            mkdir -p "$HOME/.local/bin"
+            cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
+            docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
+            docker rm "$cid"
+            redis-server --version | grep -q 'v=7.2.16'
+      - run:
+          name: Run Redis compatibility tests
+          command: |
+            mkdir -p test-results
+            env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
+              timeout --signal=TERM 15m uv run --no-sync pytest \
+              tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
+              tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
+              tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
+              tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
+              --tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
+              --cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
+      - when:
+          condition:
+            equal: ["5.3.1", << parameters.redis_py >>]
+          steps:
+            - install_codecov_cli
+            - run:
+                name: Upload Redis compatibility coverage
+                when: always
+                command: |
+                  [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
+                  codecov upload-process --disable-search --fail-on-error -f coverage.xml \
+                    -F redis-compat -C "$CIRCLE_SHA1" \
+                    -n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -3376,6 +3567,39 @@ workflows:
             parameters:
               suite: [management, database]
               mode: [replica]
+      - postgres_suite:
+          name: proxy-behavior
+          test_path: tests/proxy_behavior
+          seed: true
+          coverage_flag: lens-postgres
+          timeout_minutes: 25
+      - postgres_suite:
+          name: proxy-security
+          test_path: tests/proxy_security_tests
+          seed: true
+          timeout_minutes: 15
+      - postgres_suite:
+          name: schema-migration
+          test_path: tests/proxy_migration_tests
+          seed: false
+          timeout_minutes: 20
+      - postgres_suite:
+          name: roi-database
+          test_path: tests/integration/database/test_roi_observed.py
+          seed: false
+          coverage_flag: roi-postgres
+          timeout_minutes: 10
+      - mcp_integration:
+          name: mcp-integration
+      - redis_compat:
+          name: redis-compat-<< matrix.redis_py >>
+          matrix:
+            parameters:
+              redis_py:
+                - "5.3.1"
+                - "6.4.0"
+                - "7.4.1"
+                - "8.0.1"
   build_and_test:
     unless:
       or:
@@ -3385,7 +3609,6 @@ workflows:
     jobs:
       - using_litellm_on_windows
       - windows_release_wheel
-      - unit
       - provider_replay_harness
       - base_sdk_install
       - local_testing_part1

.circleci/scripts/classify_changes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
 
 has_client=false
 has_backend=false
@@ -10,9 +10,14 @@ has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
 has_windows_release=false
+has_redis_compat=false
 outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
+  case "$file" in
+    litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
+      has_redis_compat=true ;;
+  esac
   case "$file" in
     *.md | *.mdx) : ;;
     pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
@@ -54,6 +59,9 @@ case "$category" in
   windows-release)
     [ "$has_windows_release" = true ] && echo run || echo skip
     ;;
+  redis-compat)
+    [ "$has_redis_compat" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/path_filter.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: path_filter.sh <backend|client|provider-harness>}"
+category="${1:?usage: path_filter.sh <backend|client|provider-harness|redis-compat>}"
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 run_full() {

.circleci/scripts/run_integration.sh

@@ -72,7 +72,7 @@ export PATH="$PWD/.venv/bin:$PATH"
 export PYTHONPATH="$PWD:$PWD/tests:$PWD/tests/e2e"
 export DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:5432/circle_test"
 export REDIS_HOST=127.0.0.1 REDIS_PORT=6379
-export LITELLM_MASTER_KEY=sk-integration-master LITELLM_SALT_KEY=sk-integration-salt
+export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)" LITELLM_SALT_KEY=sk-integration-salt
 export LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True
 export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1
 export INTEGRATION_PROXY_URL=http://127.0.0.1:4000

.circleci/scripts/unit_selection.sh

@@ -1,188 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-flag="${1:?usage: unit_selection.sh <codecov flag>}"
-
-legacy_flags=(
-  caching-local
-  core-utils
-  enterprise-package
-  enterprise-routing
-  integrations
-  llm-other-providers
-  llm-vertex-ai
-  mcp-integration
-  misc
-  proxy-db-auth-checks
-  proxy-db-budgets
-  proxy-db-custom-logging
-  proxy-db-db-and-spend
-  proxy-db-endpoints-and-responses
-  proxy-db-guardrails-hooks
-  proxy-db-jwt-and-keys
-  proxy-db-key-generation
-  proxy-db-logging-misc
-  proxy-db-proxy-runtime
-  proxy-db-proxy-server-core
-  proxy-db-proxy-utils
-  proxy-extras
-  proxy-infra
-  responses-caching-types
-)
-
-legacy_paths() {
-  case "$1" in
-    caching-local) echo tests/unit/caching ;;
-    core-utils) echo tests/unit/litellm_core_utils ;;
-    enterprise-package)
-      echo tests/unit/enterprise/integrations
-      echo tests/unit/enterprise/proxy/auth
-      echo tests/unit/enterprise/proxy/guardrails
-      echo tests/unit/enterprise/proxy/hooks
-      echo tests/unit/enterprise/proxy/management_endpoints
-      echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
-      echo tests/unit/enterprise/proxy/test_liteadmin.py
-      echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
-    enterprise-routing)
-      echo tests/unit/google_genai
-      echo tests/unit/router_strategy
-      echo tests/unit/router_utils
-      echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
-      echo tests/unit/enterprise/enterprise_callbacks/send_emails
-      echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
-      echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
-      echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
-      echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
-      echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
-      echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
-      echo tests/unit/enterprise/proxy/test_enterprise_routes.py
-      echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
-      echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
-      echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
-    integrations) echo tests/unit/integrations ;;
-    llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
-    llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
-    mcp-integration)
-      echo tests/unit/experimental_mcp_client
-      echo tests/unit/proxy/_experimental/mcp_server
-      echo tests/unit/responses/mcp
-      echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
-    misc)
-      find tests/unit -maxdepth 1 -name 'test_*.py'
-      echo tests/unit/test_router
-      echo tests/unit/a2a_protocol
-      echo tests/unit/batches
-      echo tests/unit/chat_completions
-      echo tests/unit/completion_extras
-      echo tests/unit/containers
-      echo tests/unit/embeddings
-      echo tests/unit/endpoints
-      echo tests/unit/files
-      echo tests/unit/harness
-      echo tests/unit/images
-      echo tests/unit/interactions
-      echo tests/unit/messages
-      echo tests/unit/rag
-      echo tests/unit/rerank_api
-      echo tests/unit/rust_bridge
-      echo tests/unit/secret_managers
-      echo tests/unit/vector_stores
-      echo tests/unit/videos ;;
-    proxy-db-auth-checks)
-      echo tests/unit/proxy/auth/test_auth_checks.py
-      echo tests/unit/proxy/auth/test_user_api_key_auth.py
-      echo tests/unit/proxy/test_credential_slot_registry.py
-      echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
-    proxy-db-budgets)
-      echo tests/proxy_unit_tests/test_end_user_model_max_budget_enforcement.py
-      echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
-      echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
-      echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
-    proxy-db-custom-logging)
-      echo tests/unit/proxy/test_custom_callback_input.py
-      echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
-    proxy-db-db-and-spend)
-      echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
-      echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
-      echo tests/unit/proxy/db/test_update_daily_tag_spend.py
-      echo tests/unit/proxy/test_db_schema_changes.py
-      echo tests/unit/proxy/test_prisma_client_backoff_retry.py
-      echo tests/unit/proxy/test_update_spend.py
-      echo tests/unit/skills/test_skills_db.py ;;
-    proxy-db-endpoints-and-responses)
-      echo tests/unit/proxy/lens
-      echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
-      echo tests/unit/proxy/common_utils/test_check_batch_cost.py
-      echo tests/unit/proxy/common_utils/test_check_responses_cost.py
-      echo tests/unit/proxy/common_utils/test_realtime_cache.py
-      echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
-      echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
-      echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
-      echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
-      echo tests/unit/proxy/response_polling
-      echo tests/unit/proxy/test_custom_tokenizer_bug.py
-      echo tests/unit/proxy/test_get_favicon.py
-      echo tests/unit/proxy/test_get_image.py
-      echo tests/unit/proxy/test_prompt_test_endpoint.py
-      echo tests/unit/proxy/test_reducto_ocr_route.py
-      echo tests/unit/proxy/test_response_polling_pre_call_checks.py
-      echo tests/unit/proxy/test_ui_path_detection.py ;;
-    proxy-db-guardrails-hooks)
-      echo tests/unit/proxy/hooks/test_banned_keyword_list.py
-      echo tests/unit/proxy/test_proxy_setting_guardrails.py
-      echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
-    proxy-db-jwt-and-keys)
-      echo tests/unit/proxy/auth/test_jwt.py
-      echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
-      echo tests/unit/proxy/test_proxy_custom_auth.py ;;
-    proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
-    proxy-db-logging-misc)
-      echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
-      echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
-      echo tests/unit/proxy/test_proxy_reject_logging.py ;;
-    proxy-db-proxy-runtime)
-      echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
-      echo tests/unit/proxy/auth/test_proxy_routes.py
-      echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
-      echo tests/unit/proxy/test_proxy_config_unit_test.py
-      echo tests/unit/proxy/test_proxy_token_counter.py
-      echo tests/unit/proxy/test_server_root_path.py ;;
-    proxy-db-proxy-server-core)
-      echo tests/unit/proxy/test__lazy_features.py
-      echo tests/unit/proxy/test_aproxy_startup.py
-      echo tests/unit/proxy/test_proxy_server.py ;;
-    proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
-    proxy-extras) echo tests/unit/litellm_proxy_extras ;;
-    proxy-infra)
-      echo tests/unit/gateway
-      echo tests/unit/proxy/management
-      echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
-      echo tests/unit/proxy/roi_calculator ;;
-    responses-caching-types)
-      find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
-      echo tests/unit/types ;;
-    *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
-  esac
-}
-
-expand() {
-  while read -r path; do
-    if [ -d "$path" ]; then
-      find "$path" -name 'test_*.py'
-    elif [ -f "$path" ]; then
-      echo "$path"
-    else
-      echo "unit_selection.sh: $path does not exist" >&2
-      exit 1
-    fi
-  done
-}
-
-if [ "$flag" = unit ]; then
-  comm -23 \
-    <(find tests/unit -name 'test_*.py' | sort) \
-    <(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
-  exit 0
-fi
-
-legacy_paths "$flag" | expand | sort

.circleci/tests.yml

@@ -1,416 +0,0 @@
-version: 2.1
-
-commands:
-  wait_for_service:
-    parameters:
-      url:
-        type: string
-      timeout:
-        type: string
-        default: "60"
-    steps:
-      - run:
-          name: "Wait for << parameters.url >>"
-          command: |
-            TIMEOUT=<< parameters.timeout >>
-            URL="<< parameters.url >>"
-            ELAPSED=0
-            echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
-            if echo "$URL" | grep -q '^tcp://'; then
-              HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
-              PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
-              while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
-                sleep 2; ELAPSED=$((ELAPSED+2))
-                if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
-              done
-            else
-              while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
-                sleep 2; ELAPSED=$((ELAPSED+2))
-                if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
-              done
-            fi
-            echo "Service ready after ${ELAPSED}s"
-  install_uv:
-    steps:
-      - run:
-          name: Install uv (pinned 0.10.9)
-          command: |
-            curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
-            echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f  /tmp/uv-install.sh" | sha256sum -c -
-            env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
-            rm -f /tmp/uv-install.sh
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
-            export PATH="$HOME/.local/bin:$PATH"
-  install_rust:
-    steps:
-      - run:
-          name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
-          command: |
-            case "$(uname -m)" in
-              x86_64)
-                RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
-                RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
-                ;;
-              aarch64)
-                RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
-                RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
-                ;;
-              *)
-                echo "install_rust: unsupported architecture $(uname -m)" >&2
-                exit 1
-                ;;
-            esac
-            curl -sSLf -o /tmp/rustup-init \
-              "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
-            echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
-            chmod +x /tmp/rustup-init
-            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
-            rm -f /tmp/rustup-init
-            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
-            export PATH="$HOME/.cargo/bin:$PATH"
-            rustc --version
-            cargo --version
-  install_codecov_cli:
-    steps:
-      - run:
-          name: Install Codecov CLI (pinned v11.3.1)
-          when: always
-          command: |
-            curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
-            curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
-            [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d  codecov" ]
-            (cd /tmp && sha256sum -c codecov.SHA256SUM)
-            chmod +x /tmp/codecov
-            mkdir -p "$HOME/.local/bin"
-            mv /tmp/codecov "$HOME/.local/bin/codecov"
-  setup_litellm_enterprise_pip:
-    steps:
-      - run:
-          name: "Install local version of litellm-enterprise"
-          command: |
-            uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
-  setup_test_deps:
-    steps:
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
-      - run:
-          name: Generate Prisma client
-          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-  skip_unless_relevant:
-    parameters:
-      category:
-        type: string
-        default: backend
-      base_ref:
-        type: string
-        default: ""
-      pull_request_url:
-        type: string
-        default: ""
-    steps:
-      - run:
-          name: "Skip job when no << parameters.category >>-relevant files changed"
-          command: |
-            export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}"
-            export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>"
-            [ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH
-            bash .circleci/scripts/path_filter.sh << parameters.category >>
-  start_postgres:
-    parameters:
-      db_name:
-        type: string
-        default: circle_test
-      image:
-        type: string
-        default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
-    steps:
-      - run:
-          name: Start PostgreSQL
-          command: |
-            docker run -d \
-              --name postgres-db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=postgres \
-              -e POSTGRES_DB=<< parameters.db_name >> \
-              -p 5432:5432 \
-              << parameters.image >>
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-  start_redis:
-    steps:
-      - run:
-          name: Start Redis
-          command: |
-            docker run -d \
-              --name redis-cache \
-              -p 6379:6379 \
-              redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
-      - wait_for_service:
-          url: tcp://localhost:6379
-          timeout: "60"
-
-jobs:
-  unit:
-    parameters:
-      flag:
-        type: string
-        default: unit
-      shards:
-        type: integer
-        default: 6
-      workers:
-        type: integer
-        default: 4
-      dist:
-        type: string
-        default: loadscope
-      base_ref:
-        type: string
-        default: ""
-      pull_request_url:
-        type: string
-        default: ""
-      legacy_mcp_peer:
-        type: boolean
-        default: false
-      reruns:
-        type: integer
-        default: 0
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    parallelism: << parameters.shards >>
-    environment:
-      COVERAGE_CORE: sysmon
-      LITELLM_LOCAL_MODEL_COST_MAP: "True"
-    steps:
-      - checkout
-      - skip_unless_relevant:
-          base_ref: << parameters.base_ref >>
-          pull_request_url: << parameters.pull_request_url >>
-      - setup_test_deps
-      - when:
-          condition: << parameters.legacy_mcp_peer >>
-          steps:
-            - run:
-                name: Install MCP SDK1 peer
-                command: |
-                  uv venv --python 3.12 .venv-mcp-peer
-                  uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
-                  echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
-      - run:
-          name: "Run << parameters.flag >> shard"
-          no_output_timeout: 20m
-          command: |
-            mkdir -p test-results/<< parameters.flag >>
-            selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
-            [ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
-            shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
-            [ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
-            mapfile -t files < <(printf '%s\n' "${shard}")
-            xdist_args=()
-            if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
-            rerun_args=(-p no:rerunfailures)
-            if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
-            test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
-            if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
-            set +e
-            env -i "${test_env[@]}" \
-              uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
-            status=$?
-            set -e
-            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
-            exit "$status"
-      - install_codecov_cli
-      - run:
-          name: Upload coverage
-          when: always
-          command: |
-            [ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; }
-            codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test-results
-      - store_artifacts:
-          path: coverage.xml
-  documentation:
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    steps:
-      - checkout
-      - setup_test_deps
-      - run:
-          name: Checkout litellm-docs
-          command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website
-      - run:
-          name: Run documentation validation
-          command: |
-            uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
-            uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
-            uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
-            uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
-  integration:
-    parameters:
-      suite:
-        type: string
-      base_ref:
-        type: string
-        default: ""
-      pull_request_url:
-        type: string
-        default: ""
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_unless_relevant:
-          base_ref: << parameters.base_ref >>
-          pull_request_url: << parameters.pull_request_url >>
-      - setup_test_deps
-      - start_postgres:
-          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
-      - start_redis
-      - run:
-          name: Run owned integration contracts
-          command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
-          no_output_timeout: 15m
-      - run:
-          name: Stop owned database and Redis
-          when: always
-          command: |
-            mkdir -p test-results/integration-<< parameters.suite >>
-            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
-            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
-            docker rm -f postgres-db redis-cache
-            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test-results
-
-workflows:
-  tests:
-    when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_"))
-    jobs:
-      - unit:
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-<< matrix.flag >>
-          shards: 1
-          workers: 2
-          reruns: 2
-          matrix:
-            parameters:
-              flag: [caching-local, proxy-extras, enterprise-routing]
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-mcp-integration
-          flag: mcp-integration
-          shards: 1
-          workers: 2
-          legacy_mcp_peer: true
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-<< matrix.flag >>
-          shards: 1
-          reruns: 2
-          matrix:
-            parameters:
-              flag:
-                - enterprise-package
-                - proxy-infra
-                - responses-caching-types
-                - proxy-db-auth-checks
-                - proxy-db-jwt-and-keys
-                - proxy-db-proxy-server-core
-                - proxy-db-proxy-runtime
-                - proxy-db-custom-logging
-                - proxy-db-logging-misc
-                - proxy-db-db-and-spend
-                - proxy-db-guardrails-hooks
-                - proxy-db-budgets
-                - proxy-db-endpoints-and-responses
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-llm-vertex-ai
-          flag: llm-vertex-ai
-          shards: 2
-          workers: 1
-          reruns: 2
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-llm-other-providers
-          flag: llm-other-providers
-          shards: 3
-          reruns: 2
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-core-utils
-          flag: core-utils
-          shards: 2
-          reruns: 1
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-integrations
-          flag: integrations
-          shards: 2
-          reruns: 3
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-misc
-          flag: misc
-          shards: 2
-          reruns: 2
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-proxy-db-proxy-utils
-          flag: proxy-db-proxy-utils
-          shards: 1
-          reruns: 2
-          dist: worksteal
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - unit:
-          name: unit-proxy-db-key-generation
-          flag: proxy-db-key-generation
-          shards: 1
-          workers: 0
-          reruns: 2
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
-      - documentation
-      - integration:
-          name: integration-<< matrix.suite >>
-          matrix:
-            parameters:
-              suite: [sdk]
-          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
-          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>

.gitguardian.yaml

@@ -84,10 +84,6 @@ secret:
     - name: Langfuse test credentials in test_completion
       match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
     
-    # Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
-    - name: Test password in e2e test fixtures
-      match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
-    
     # === Preventive patterns for test keys (pattern-based) ===
     
     # Test API keys (124 instances across 45 files)
@@ -108,4 +104,3 @@ secret:
 
     - name: Short fake sk keys (1–9 digits only)
       match: \bsk-\d{1,9}\b
-

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -150,7 +150,7 @@ For each e2e test you added or changed, list the manual steps a reviewer can fol
 Example checklists:
 
 - tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py::TestKeyRateLimits::test_rpm_limit_blocks_over_limit - a key allowed 2 requests a minute serves exactly 2 and refuses the 3rd
-  - [ ] Generate a limited key: curl -X POST http://localhost:4000/key/generate -H "Authorization: Bearer sk-1234" -d '{"rpm_limit": 2}'
+  - [ ] Generate a limited key: curl -X POST http://localhost:4000/key/generate -H "Authorization: Bearer $LITELLM_MASTER_KEY" -d '{"rpm_limit": 2}'
   - [ ] Send three /v1/chat/completions requests with that key inside one minute
   - [ ] Expect the first two to return 200 and the third to return 429 naming the rpm limit
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
@@ -164,4 +164,3 @@ Example checklists:
 ## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
-

.github/PULL_REQUEST_TEMPLATE/rust.md

@@ -0,0 +1 @@
+<!-- Write the final PR description as concise bullets only -->

.github/actions/rust-bridge/action.yml

@@ -0,0 +1,19 @@
+name: Set up the Rust bridge
+description: Select the shared Rust bridge artifact or the Cargo cache
+inputs:
+  artifact:
+    description: Rust bridge artifact name
+    required: false
+    default: ""
+runs:
+  using: composite
+  steps:
+    - name: Restore the Cargo build cache
+      if: inputs.artifact == ''
+      uses: ./.github/actions/cache-cargo-build
+    - name: Download the Rust bridge artifact
+      if: inputs.artifact != ''
+      uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e
+      with:
+        name: ${{ inputs.artifact }}
+        path: rust-bridge-dist

.github/ci-coverage-allowlist.yml

@@ -21,7 +21,7 @@ test_paths:
       Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
       glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
       not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
-      Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
+      Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in
       test_caching.py by node ID. It does not run that file's other tests.
       The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
       run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider

.github/e2e-stack/down.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -uo pipefail
-
-STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
-
-for pid_file in "${STACK_DIR}"/pids/*.pid; do
-  [[ -f "${pid_file}" ]] || continue
-  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
-  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
-  rm -f "${pid_file}"
-done
-
-for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do
-  docker rm -f "${container}" >/dev/null 2>&1
-done
-
-exit 0

.github/e2e-stack/redact_output.py

@@ -1,83 +0,0 @@
-import argparse
-import os
-import sys
-from functools import reduce
-from pathlib import Path
-from typing import Final
-from xml.sax.saxutils import escape
-
-from pydantic import JsonValue, TypeAdapter, ValidationError
-from secrets_to_env import MIN_MASKED_LENGTH
-
-REDACTED: Final = "***"
-json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
-
-
-def string_leaves(node: JsonValue) -> tuple[str, ...]:
-    match node:
-        case str():
-            return (node,)
-        case list():
-            return tuple(leaf for child in node for leaf in string_leaves(child))
-        case dict():
-            return tuple(leaf for child in node.values() for leaf in string_leaves(child))
-    return ()
-
-
-def field_lines(value: str) -> tuple[str, ...]:
-    try:
-        return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
-    except ValidationError:
-        return ()
-
-
-def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
-    values: Final = frozenset(
-        line.split("=", 1)[1].strip().strip("'")
-        for path in values_files
-        for line in path.read_text().splitlines()
-        if "=" in line
-    )
-    texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
-    renderings: Final = frozenset(
-        rendering
-        for text in texts
-        if len(text) >= MIN_MASKED_LENGTH
-        for rendering in (text, escape(text), escape(text, {'"': """}))
-    )
-    return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
-
-
-def redact(text: str, values: tuple[str, ...]) -> str:
-    return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
-
-
-def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
-    target: Final = out_dir / source.name
-    with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
-        _ = handle.write(redact(source.read_text(errors="replace"), values))
-
-
-def main() -> int:
-    parser: Final = argparse.ArgumentParser()
-    _ = parser.add_argument("--values", action="append", type=Path, required=True)
-    _ = parser.add_argument("--out", type=Path, required=True)
-    _ = parser.add_argument("files", nargs="*", type=Path)
-    args: Final = parser.parse_args()
-    values_files: Final = tuple(args.values)
-    out_dir: Final[Path] = args.out
-    sources: Final = tuple(args.files)
-    try:
-        values: Final = masked_values(values_files)
-        out_dir.mkdir(mode=0o700, exist_ok=True)
-        for source in sources:
-            write_redacted(source, out_dir, values)
-    except OSError as error:
-        _ = sys.stderr.write(f"could not redact {error.filename}\n")
-        return 1
-    _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/e2e-stack/secrets_to_env.py

@@ -1,55 +0,0 @@
-import os
-import re
-import sys
-from pathlib import Path
-from typing import Final
-
-from pydantic import TypeAdapter, ValidationError
-
-secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
-ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
-MIN_MASKED_LENGTH: Final = 8
-ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
-
-
-def main() -> int:
-    env_path: Final = Path(sys.argv[1])
-    try:
-        secrets: Final = {
-            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
-        }
-    except (ValidationError, UnicodeError):
-        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
-        return 1
-    unusable: Final = tuple(
-        key
-        for key, value in secrets.items()
-        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
-    )
-    if unusable:
-        _ = sys.stderr.write(
-            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
-        )
-        return 1
-    if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
-        _ = sys.stdout.write(
-            "".join(
-                f"::add-mask::{value.replace('%', '%25')}\n"
-                for value in secrets.values()
-                if len(value) >= MIN_MASKED_LENGTH
-            )
-        )
-        sys.stdout.flush()
-    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
-    try:
-        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
-            os.fchmod(handle.fileno(), 0o600)
-            _ = handle.write("\n".join(lines) + "\n")
-    except OSError:
-        _ = sys.stderr.write("could not write the environment file\n")
-        return 1
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -1,52 +0,0 @@
-import re
-import sys
-from typing import Final
-
-SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
-UNSUPPORTED: Final = re.compile(
-    r"^tests/e2e/(ui|claude_code|load|migrations)/"
-    r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$"
-    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
-    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
-    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
-    r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
-    r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
-    r"|^tests/e2e/logging/test_s3_log_e2e\.py$"
-    r"|^tests/e2e/secret_manager/"
-)
-HARNESS: Final = re.compile(
-    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
-    r"|^tests/e2e/idp_realm\.json$"
-    r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
-    r"|^tests/e2e/coverage_registry/management_cases\.py$"
-    r"|^tests/e2e/gateway/"
-    r"|^\.github/e2e-stack/"
-    r"|^\.github/workflows/test-e2e-changed\.yml$"
-)
-UNEXPANDED: Final = re.compile(r"[*?\[]")
-
-
-def is_selectable(path: str) -> bool:
-    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
-
-
-def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
-    direct: Final = frozenset(path for path in changed if is_selectable(path))
-    harness_changed: Final = any(HARNESS.match(path) for path in changed)
-    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
-    return tuple(sorted(direct | canary_tests))
-
-
-def main() -> int:
-    canary: Final = tuple(sys.argv[1:])
-    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
-    if unexpanded:
-        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
-        return 1
-    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
-    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/e2e-stack/up.sh

@@ -1,231 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-umask 077
-
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
-CERTS_DIR="${STACK_DIR}/certs"
-LOGS_DIR="${STACK_DIR}/logs"
-PIDS_DIR="${STACK_DIR}/pids"
-
-POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
-VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
-JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
-NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
-
-LB_PORT="${E2E_LB_PORT:-4000}"
-GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
-GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
-BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
-REDIS_PORT="${E2E_REDIS_PORT:-6379}"
-DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
-DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
-DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
-DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
-DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
-JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
-JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
-JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
-KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
-
-MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
-
-mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
-chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
-chmod 755 "${CERTS_DIR}"
-
-log() { printf 'e2e-stack: %s\n' "$*"; }
-
-port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
-
-wait_for() {
-  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
-  until eval "${check}"; do
-    if ((SECONDS >= deadline)); then
-      log "timed out waiting for ${label}"
-      exit 1
-    fi
-    sleep 2
-  done
-  log "${label} is up"
-}
-
-if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
-  set -a
-  source "${REPO_ROOT}/tests/e2e/.env"
-  set +a
-fi
-
-if [[ -z "${DD_API_KEY:-}" ]]; then
-  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
-  exit 1
-fi
-export DD_SITE="${DD_SITE:-datadoghq.com}"
-
-if ! port_open "${DATABASE_PORT}"; then
-  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
-    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
-    "${POSTGRES_IMAGE}" >/dev/null
-fi
-wait_for "postgres" "port_open ${DATABASE_PORT}"
-
-if ! port_open "${JAEGER_QUERY_PORT}"; then
-  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
-    "${JAEGER_IMAGE}" >/dev/null
-fi
-wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
-
-openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
-openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
-  -subj "/CN=litellm-e2e-ca" \
-  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
-  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
-openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
-openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
-openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
-  -CAcreateserial -days 7 -sha256 \
-  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
-  -out "${CERTS_DIR}/server.crt" 2>/dev/null
-chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
-
-CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
-cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
-
-docker rm -f e2e-valkey >/dev/null 2>&1 || true
-docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
-  "${VALKEY_IMAGE}" valkey-server \
-  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
-  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
-  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
-VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
-wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
-${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
-wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
-
-CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
-CONFIG_PATH="${CONFIG_SOURCE}"
-if [[ "${REDIS_PORT}" != "6379" ]]; then
-  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
-  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
-fi
-
-SERVER_ENV=(
-  "LITELLM_MASTER_KEY=${MASTER_KEY}"
-  "DATABASE_HOST=${DATABASE_HOST}"
-  "DATABASE_PORT=${DATABASE_PORT}"
-  "DATABASE_USER=${DATABASE_USER}"
-  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
-  "DATABASE_NAME=${DATABASE_NAME}"
-  "DISABLE_SCHEMA_UPDATE=true"
-  "REDIS_HOST=127.0.0.1"
-  "REDIS_PORT=${REDIS_PORT}"
-  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
-  "CONFIG_FILE_PATH=${CONFIG_PATH}"
-  "STORE_MODEL_IN_DB=True"
-  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
-  "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
-  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
-  "PYTHONPATH=${REPO_ROOT}"
-  "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
-  "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
-  "JWT_AUDIENCE=litellm-e2e"
-)
-if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
-  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
-  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
-fi
-
-cd "${REPO_ROOT}"
-
-env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh
-
-log "running migrations"
-env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
-
-start_server() {
-  local name="$1"; shift
-  env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
-  echo $! > "${PIDS_DIR}/${name}.pid"
-}
-
-if [[ "$(uname)" == "Linux" ]]; then
-  NGINX_UPSTREAM_HOST=127.0.0.1
-  NGINX_DOCKER_ARGS=(--network host)
-else
-  NGINX_UPSTREAM_HOST=host.docker.internal
-  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
-fi
-
-cat > "${STACK_DIR}/nginx.conf" <<EOF
-events {}
-http {
-  map \$http_upgrade \$connection_upgrade {
-    default upgrade;
-    '' close;
-  }
-  upstream litellm_gateways {
-    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
-    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
-  }
-  server {
-    listen ${LB_PORT};
-    client_max_body_size 100m;
-    location / {
-      proxy_pass http://litellm_gateways;
-      proxy_http_version 1.1;
-      proxy_set_header Host \$host;
-      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto \$scheme;
-      proxy_set_header Upgrade \$http_upgrade;
-      proxy_set_header Connection \$connection_upgrade;
-      proxy_buffering off;
-      proxy_read_timeout 600s;
-      proxy_send_timeout 600s;
-    }
-  }
-  server {
-    listen ${JAEGER_OTLP_TLS_PORT} ssl;
-    ssl_certificate /certs/server.crt;
-    ssl_certificate_key /certs/server.key;
-    client_max_body_size 100m;
-    location / {
-      proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
-    }
-  }
-}
-EOF
-
-docker rm -f e2e-nginx >/dev/null 2>&1 || true
-docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
-  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
-  -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
-
-wait_for "Jaeger OTLP TLS listener" \
-  "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
-
-start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
-start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
-start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
-
-wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
-wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
-wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
-wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
-
-cat > "${STACK_DIR}/stack.env" <<EOF
-LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
-LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
-LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
-LITELLM_MASTER_KEY=${MASTER_KEY}
-REDIS_HOST=127.0.0.1
-REDIS_PORT=${REDIS_PORT}
-E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
-E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
-E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
-E2E_KEYCLOAK_ADMIN_USER=admin
-E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
-SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
-DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
-EOF
-
-log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/scripts/assert_ci_coverage.py

@@ -9,7 +9,6 @@ import sys
 import warnings
 from collections.abc import Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
-from types import MappingProxyType
 from typing import Final
 
 import yaml
@@ -22,11 +21,12 @@ TESTS_ROOT = REPO_ROOT / "tests"
 
 ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
 PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
-TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
+TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"})
 DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
 TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
 IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
 TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
+IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
 DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
 COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
 GLOB_CHARS = frozenset("*?")
@@ -72,6 +72,17 @@ class Scalar:
     value: str
 
 
+@dataclass(frozen=True, slots=True)
+class Selection:
+    included: frozenset[str]
+    ignored: frozenset[str]
+
+    def covers(self, relative_path: str) -> bool:
+        return any(_token_covers(token, relative_path) for token in self.included) and not any(
+            _token_covers(token, relative_path) for token in self.ignored
+        )
+
+
 @dataclass(frozen=True, slots=True)
 class Finding:
     subject: str
@@ -111,44 +122,32 @@ def _uncommented(value: str) -> str:
     return COMMENT_RE.sub("", value)
 
 
-def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
-    return frozenset(
-        match.group(0).rstrip("/")
+def _selection_for_scalar(scalar: Scalar) -> Selection:
+    text: Final = _uncommented(scalar.value)
+    ignored: Final = frozenset().union(
+        *(
+            frozenset(token.rstrip("/") for token in TEST_TOKEN_RE.findall(ignored_argument))
+            for ignored_argument in IGNORE_ARG_RE.findall(text)
+        )
+    )
+    included: Final = frozenset(
+        match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(IGNORE_ARG_RE.sub("", text))
+    )
+    return Selection(included=included, ignored=ignored)
+
+
+def _invoked_selections(scalars: Iterable[Scalar]) -> tuple[Selection, ...]:
+    selected_scalars: Final = tuple(
+        scalar
         for scalar in scalars
         if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
-        for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
     )
+    selections: Final = tuple(_selection_for_scalar(scalar) for scalar in selected_scalars)
+    return tuple(selection for selection in selections if selection.included)
 
 
-SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;")
-
-
-def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]:
-    script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
-    if not script.is_file():
-        return MappingProxyType({})
-    text: Final = _uncommented(script.read_text())
-    return MappingProxyType(
-        {
-            label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body))
-            for label, body in SELECTION_ARM_RE.findall(text)
-        }
-    )
-
-
-def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
-    return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens)
-
-
-def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
-    return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value)
-
-
-def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]:
-    wired: Final = _wired_unit_flags(scalars)
-    return _invoked_test_tokens(scalars) | frozenset(
-        token for label, tokens in arms.items() if label in wired for token in tokens
-    )
+def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset().union(*(selection.included for selection in _invoked_selections(scalars)))
 
 
 def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
@@ -211,11 +210,12 @@ def _dockerfiles() -> tuple[str, ...]:
     )
 
 
-def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+def _uncovered_tests(allowlist: Allowlist, selections: tuple[Selection, ...]) -> tuple[Finding, ...]:
     uncovered = tuple(
         relative_path
         for relative_path in _test_files()
-        if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
+        if not any(selection.covers(relative_path) for selection in selections)
+        and not allowlist.covers_test(relative_path)
     )
     directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
     return tuple(
@@ -505,7 +505,7 @@ def _check_slices() -> int:
 
 
 def _check_shards() -> int:
-    findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms()))
+    findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
     if findings:
         _report(
             "test directories and files that no shard claims",
@@ -568,6 +568,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
     browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes)
     circle_path: Final = repo_root / ".circleci/config.yml"
     circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
+    circle_test_path_tokens: Final = _invoked_test_tokens(
+        scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path"
+    )
     steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
     invoked: Final = any(
         ".circleci/scripts/run_integration.sh" in scalar.value
@@ -609,9 +612,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
             if any(_token_covers(token, path) for token in gha_tokens)
         )
         + tuple(
-            Finding(path, "GitHub-owned integration contract has no invoking workflow")
+            Finding(path, "GitHub-owned integration contract has no invoking job")
             for path in sorted(github_files)
-            if not any(_token_covers(token, path) for token in gha_tokens)
+            if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens)
         )
         + tuple(
             Finding(path, "GitHub-owned integration file is missing")
@@ -675,7 +678,11 @@ def main() -> int:
 
     integration_paths, ownership_findings = _integration_ownership()
     test_findings = (
-        _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
+        _uncovered_tests(
+            allowlist,
+            _invoked_selections(scalars)
+            + (Selection(included=integration_paths, ignored=frozenset()),),
+        )
         + ownership_findings
     )
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))

.github/scripts/verify_linux_native_wheel.py

@@ -214,7 +214,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 45_000_000
+    native_size_limit: Final = 48_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),

.github/workflows/_test-unit-base.yml

@@ -13,16 +13,6 @@ on:
           have its path existence-checked like any other token.
         required: true
         type: string
-      unit-flag:
-        description: >-
-          Codecov flag of the `.circleci/tests.yml` job that now owns part of
-          this shard. The shard also runs the files
-          `.circleci/scripts/unit_selection.sh` lists for the flag, on every
-          event, because the CircleCI pipeline is manual-only while the tests
-          migrate.
-        required: false
-        type: string
-        default: ""
       workers:
         description: "Number of pytest-xdist workers"
         required: false
@@ -79,12 +69,11 @@ on:
         description: "Unique name for the coverage artifact (must be unique per run)"
         required: true
         type: string
-      legacy-mcp-peer:
-        description: "Install the isolated SDK1 peer for MCP compatibility tests"
+      rust-bridge-artifact:
+        description: "Prebuilt editable Rust bridge artifact"
         required: false
-        type: boolean
-        default: false
-
+        type: string
+        default: ""
 permissions:
   contents: read
 
@@ -134,25 +123,29 @@ jobs:
         timeout-minutes: 5
         uses: ./.github/actions/cache-uv-downloads
 
-      - name: Cache the Rust build
+      - name: Set up the Rust build
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 5
-        uses: ./.github/actions/cache-cargo-build
+        uses: ./.github/actions/rust-bridge
+        with:
+          artifact: ${{ inputs.rust-bridge-artifact }}
 
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         env:
-          LEGACY_MCP_PEER: ${{ inputs.legacy-mcp-peer }}
+          RUST_BRIDGE_ARTIFACT: ${{ inputs.rust-bridge-artifact }}
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
-          uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
-          if [ "$LEGACY_MCP_PEER" = "true" ]; then
-            uv venv --python "${UV_PYTHON}" .venv-mcp-peer
-            uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
-            echo "MCP_TEST_PEER_PYTHON=$GITHUB_WORKSPACE/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
+          if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
+          else
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
+            uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
+            cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
+            uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
           fi
+          uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'
@@ -171,7 +164,6 @@ jobs:
         timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
-          UNIT_FLAG: ${{ inputs.unit-flag }}
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
@@ -181,9 +173,6 @@ jobs:
         run: |
           echo "has-coverage=false" >> "$GITHUB_OUTPUT"
           selection="${TEST_PATH}"
-          if [ -n "${UNIT_FLAG}" ]; then
-            selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')"
-          fi
           if [ -z "${selection// /}" ]; then
             echo "shard selection is empty; nothing to run"
             exit 0

.github/workflows/image-scan.yml

@@ -9,10 +9,12 @@ on:
     paths:
       - Dockerfile
       - docker/Dockerfile.non_root
+      - docker/Dockerfile.database
       - migrations/Dockerfile
       - migrations/run.py
       - gateway/Dockerfile
       - gateway/main.py
+      - gateway/routes/allowlist.py
       - backend/Dockerfile
       - backend/main.py
       - deploy/lens/**
@@ -21,6 +23,10 @@ on:
       - docker/component_entrypoint.sh
       - docker/entrypoint.sh
       - litellm/proxy/prisma_migration.py
+      - litellm/proxy/admin_mcp.py
+      - litellm/proxy/proxy_server.py
+      - backend/routes/allowlist.py
+      - pyproject.toml
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
@@ -142,6 +148,9 @@ jobs:
       - name: Build runtime image
         run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
 
+      - name: Tag the cached builder for Admin MCP schema setup
+        run: docker build --target builder -f docker/Dockerfile.non_root -t litellm-admin-mcp-schema:${{ github.sha }} .
+
       # The prisma bake must migrate a fresh DB with no egress as an arbitrary
       # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
       # `docker run` as the default uid with network hides a broken bake because
@@ -155,9 +164,10 @@ jobs:
       - name: Verify offline migration as a non-root uid
         env:
           LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
+          LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
 
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
@@ -176,21 +186,28 @@ jobs:
             --output table
 
   runtime-image:
-    name: runtime-image
+    name: runtime-image (${{ matrix.dockerfile }})
     runs-on: ubuntu-latest
     if: >-
       github.event_name != 'pull_request' ||
       github.event.pull_request.head.repo.full_name == github.repository
-    timeout-minutes: 30
+    timeout-minutes: 45
     permissions:
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        dockerfile: [Dockerfile, docker/Dockerfile.database]
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
 
       - name: Build runtime image
-        run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+        run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f "${{ matrix.dockerfile }}" -t litellm-runtime-scan:${{ github.sha }} .
+
+      - name: Tag the cached builder for Admin MCP schema setup
+        run: docker build --target builder -f "${{ matrix.dockerfile }}" -t litellm-admin-mcp-schema:${{ github.sha }} .
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -200,11 +217,13 @@ jobs:
       - name: Verify offline migration as a non-root uid
         env:
           LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+          LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
 
       - name: Verify the bundled Lens Compose installation and restart
+        if: matrix.dockerfile == 'Dockerfile'
         env:
           LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
         run: bash tests/e2e/migrations/lens_compose_smoke.sh
@@ -266,9 +285,10 @@ jobs:
         env:
           LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
           LITELLM_COMPONENT_PORT: "4000"
+          LITELLM_IMAGE_COMPONENT: gateway
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
 
   ui-image:
     name: ui-image
@@ -316,6 +336,9 @@ jobs:
       - name: Build backend image
         run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
 
+      - name: Tag the cached builder for Admin MCP schema setup
+        run: docker build --target builder -f backend/Dockerfile -t litellm-admin-mcp-schema:${{ github.sha }} .
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -324,7 +347,9 @@ jobs:
       - name: Verify the backend serves offline as a non-root uid
         env:
           LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
+          LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
           LITELLM_COMPONENT_PORT: "4001"
+          LITELLM_IMAGE_COMPONENT: backend
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_admin_mcp.py -v

.github/workflows/lens-worker.yml

@@ -6,12 +6,14 @@ on:
     paths:
       - deploy/lens/**
       - litellm/proxy/lens/**
+      - tests/proxy_behavior/lens/**
       - .github/workflows/lens-worker.yml
   push:
     branches: [main]
     paths:
       - deploy/lens/**
       - litellm/proxy/lens/**
+      - tests/proxy_behavior/lens/**
       - .github/workflows/lens-worker.yml
   workflow_dispatch:
 
@@ -27,6 +29,7 @@ jobs:
     permissions:
       contents: read
       packages: write
+      id-token: write
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -54,12 +57,66 @@ jobs:
           with trace_store() as store:
               assert store.count() == 0
           '
-      - name: Verify recovery after temporary storage fills
+      - name: Prepare test-only coverage tool
+        run: |
+          coverage_directory=$(mktemp -d "$RUNNER_TEMP/lens-coverage.XXXXXX")
+          curl --fail --silent --show-error --location \
+            https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl \
+            --output "$coverage_directory/coverage.whl"
+          printf '%s  %s\n' 8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1 \
+            "$coverage_directory/coverage.whl" | sha256sum --check
+          chmod 777 "$coverage_directory"
+          echo "LENS_COVERAGE_DIRECTORY=$coverage_directory" >> "$GITHUB_ENV"
+      - name: Verify confined Python execution
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
+            -v "$PWD/tests/proxy_behavior/lens/worker_python_smoke.py:/app/python_smoke.py:ro" \
+            -v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
+            -v "$LENS_COVERAGE_DIRECTORY:/coverage" \
+            -e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
+            --entrypoint python lens-worker:${{ github.sha }} \
+            -m coverage run --data-file=/coverage/.coverage.python /app/python_smoke.py
+      - name: Verify workspace investigation and live review output
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
+            -v "$PWD/tests/proxy_behavior/lens/worker_context_smoke.py:/app/context_smoke.py:ro" \
+            -v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
+            -v "$LENS_COVERAGE_DIRECTORY:/coverage" \
+            -e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
+            --entrypoint python lens-worker:${{ github.sha }} \
+            -m coverage run --data-file=/coverage/.coverage.context /app/context_smoke.py
+      - name: Verify default workspace recovery after Python scratch storage fills
         run: |
           docker run --rm --network none --read-only --cap-drop ALL \
             --tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
             -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
-            --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
+            -v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
+            -v "$LENS_COVERAGE_DIRECTORY:/coverage" \
+            -e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
+            --entrypoint python lens-worker:${{ github.sha }} \
+            -m coverage run --data-file=/coverage/.coverage.storage /app/storage_smoke.py
+      - name: Map native worker coverage to repository sources
+        if: always() && env.LENS_COVERAGE_DIRECTORY != ''
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --security-opt no-new-privileges -w /workspace \
+            -v "$PWD/litellm/proxy/lens:/workspace/litellm/proxy/lens:ro" \
+            -v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
+            -v "$LENS_COVERAGE_DIRECTORY:/coverage" \
+            -e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
+            --entrypoint /bin/sh lens-worker:${{ github.sha }} \
+            -c 'python -m coverage combine && python -m coverage xml'
+      - name: Upload native worker coverage
+        if: always() && env.LENS_COVERAGE_DIRECTORY != ''
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
+        with:
+          use_oidc: true
+          files: ${{ env.LENS_COVERAGE_DIRECTORY }}/lens-worker.xml
+          root_dir: ${{ github.workspace }}
+          flags: lens-worker
+          fail_ci_if_error: false
       - name: Publish versioned Lens worker
         if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
         env:

.github/workflows/test-code-quality.yml

@@ -95,7 +95,9 @@ jobs:
       - name: test_e2e_metadata
         env:
           PYTHONPATH: tests/e2e
-        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
+        run: |
+          export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
+          uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
 
       - name: Check merge smoke harness
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
@@ -163,6 +165,12 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_no_publicly_known_master_key
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
+
+      - name: test_check_no_publicly_known_master_key
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
+
       - name: check_unbounded_in_lists (fails on findings not in the baseline)
         run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
 

.github/workflows/test-e2e-changed.yml

@@ -1,265 +0,0 @@
-name: e2e-changed-tests
-
-on:
-  pull_request:
-
-concurrency:
-  group: e2e-changed-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-permissions: {}
-
-jobs:
-  detect:
-    name: Detect changed e2e tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    permissions:
-      contents: read
-      pull-requests: read
-    outputs:
-      tests: ${{ steps.changed.outputs.tests }}
-      any: ${{ steps.changed.outputs.any }}
-    steps:
-      - name: Checkout the selector and the canary suite
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: |
-            .github/e2e-stack
-            tests/e2e/access_control
-            tests/e2e/management/test_jwt_management_e2e.py
-            tests/e2e/other/test_jwt_auth_e2e.py
-          persist-credentials: false
-          ref: ${{ github.sha }}
-
-      - name: List the e2e test files this PR added or modified
-        id: changed
-        env:
-          GH_TOKEN: ${{ github.token }}
-          REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
-            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
-            | grep -Fxq "${HEAD_SHA}"
-          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
-            --jq '.[] | select(.status != "removed") | .filename')"
-          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
-          tests="$(printf '%s\n' "${files}" \
-            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \
-                tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)"
-          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
-          if [ -n "${tests}" ]; then
-            echo "any=true" >> "${GITHUB_OUTPUT}"
-            echo "selected e2e tests: ${tests}"
-          else
-            echo "any=false" >> "${GITHUB_OUTPUT}"
-            echo "no changed e2e test files supported by this stack; nothing to run"
-          fi
-
-  run:
-    name: Run changed e2e tests against the stage-mirror stack
-    needs: detect
-    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    environment: e2e-changed
-    permissions:
-      contents: read
-      id-token: write
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: litellm
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U litellm"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-      jaeger:
-        image: jaegertracing/jaeger:2.10.0
-        ports:
-          - 4318:4318
-          - 16686:16686
-    steps:
-      - name: Validate configuration
-        env:
-          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
-        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
-
-      - name: Checkout
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ github.sha }}
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.13"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache the Rust build
-        uses: ./.github/actions/cache-cargo-build
-
-      - name: Install dependencies
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen \
-            --extra proxy --extra proxy-runtime --extra extra_proxy \
-            --extra semantic-router --extra bedrock-realtime \
-            --group ci --group proxy-dev --group e2e-dev
-          uv pip install "pipecat-ai[openai]==1.4.0"
-
-      - name: Cache Prisma binaries
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Install Playwright chromium
-        run: uv run --no-sync playwright install --with-deps chromium
-
-      - name: Configure AWS credentials
-        id: aws
-        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
-        with:
-          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
-          aws-region: us-east-1
-          role-session-name: litellm-e2e-changed-${{ github.run_id }}
-          role-duration-seconds: 900
-          output-env-credentials: false
-          output-credentials: true
-
-      - name: Fetch provider credentials from AWS Secrets Manager
-        env:
-          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
-          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
-          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          umask 077
-          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
-            --query SecretString --output text \
-            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
-          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
-            --query SecretString --output text \
-            | jq -R -s '{"LITELLM_LICENSE": .}' \
-            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
-
-      - name: Boot the stage-mirror stack
-        id: boot
-        run: |
-          umask 077
-          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
-            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
-            exit 1
-          fi
-
-      - name: Export stack environment
-        run: |
-          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
-          echo "::add-mask::${master_key}"
-          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
-
-      - name: Run the selected tests three times
-        env:
-          TESTS: ${{ needs.detect.outputs.tests }}
-          E2E_FIXTURE_MODE: live
-          E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
-          E2E_OWNED_GATEWAY: '1'
-          COLUMNS: '400'
-        run: |
-          umask 077
-          read -r -a test_files <<< "${TESTS}"
-          for pass in 1 2 3; do
-            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
-            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
-            echo "::group::pass ${pass} of 3"
-            set +e
-            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
-              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
-            status=$?
-            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
-            verified=$?
-            set -e
-            grep -E '^(FAILED|ERROR) ' "${log}" || true
-            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
-            echo "::endgroup::"
-            if [ "${status}" = "5" ]; then
-              echo "::error::the selected files collected no runnable tests, so nothing was verified"
-              exit 1
-            fi
-            if [ "${status}" != "0" ]; then
-              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
-              exit "${status}"
-            fi
-            if [ "${verified}" != "0" ]; then
-              echo "::error::pass ${pass} of 3 did not verify every selected file"
-              exit 1
-            fi
-            echo "pass ${pass} of 3 passed"
-          done
-
-      - name: Redact the pytest output
-        if: always() && steps.boot.outcome == 'success'
-        run: |
-          umask 077
-          shopt -s nullglob
-          uv run --no-sync python .github/e2e-stack/redact_output.py \
-            --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
-            --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
-
-      - name: Keep the redacted pytest output
-        if: always() && steps.boot.outcome == 'success'
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: e2e-changed-pytest-output-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/e2e-redacted
-          retention-days: 14
-          if-no-files-found: ignore
-
-      - name: Stop the stack
-        if: always() && steps.boot.outcome != 'skipped'
-        run: bash .github/e2e-stack/down.sh
-
-      - name: Remove credentials and raw output
-        if: always()
-        run: |
-          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
-          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
-
-  gate:
-    name: e2e-changed-tests
-    needs: [detect, run]
-    if: always()
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Require three successful passes when tests changed
-        env:
-          DETECT_RESULT: ${{ needs.detect.result }}
-          ANY_TESTS: ${{ needs.detect.outputs.any }}
-          RUN_RESULT: ${{ needs.run.result }}
-        run: |
-          if [ "${DETECT_RESULT}" != "success" ]; then
-            echo "::error::changed-test detection did not succeed"
-            exit 1
-          fi
-          if [ "${ANY_TESTS}" = "false" ]; then
-            echo "no changed e2e test files supported by this stack; nothing to run"
-            exit 0
-          fi
-          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
-            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
-            exit 1
-          fi

.github/workflows/test-mcp-dependency-resolution.yml

@@ -5,6 +5,27 @@ on:
     branches:
       - main
       - "litellm_**"
+    paths:
+      - "**/pyproject.toml"
+      - "uv.lock"
+      - "uv.toml"
+      - ".python-version"
+      - "rust-toolchain.toml"
+      - "litellm-rust/**"
+      - "litellm/__init__.py"
+      - "litellm/proxy/proxy_server.py"
+      - "litellm/**/*mcp*"
+      - "litellm/**/*mcp*/**"
+      - "litellm/integrations/arize/**"
+      - "scripts/check_mcp_sdk_install.py"
+      - "tests/base_sdk_tests/**"
+      - ".github/workflows/test-mcp-dependency-resolution.yml"
+      - ".github/actions/detect-changes/**"
+      - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
+      - ".github/scripts/detect_changes.sh"
+      - ".github/scripts/uv_sync_with_retries.sh"
+      - ".circleci/scripts/classify_changes.sh"
 
 permissions:
   contents: read

.github/workflows/test-postgres.yml

@@ -1,185 +0,0 @@
-name: "Postgres Tests"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "litellm_**"
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  postgres:
-    name: ${{ matrix.shard }}
-    runs-on: ubuntu-latest
-    timeout-minutes: ${{ matrix.job-timeout-minutes }}
-    permissions:
-      contents: read
-      id-token: write
-
-    services:
-      postgres:
-        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - shard: roi-database
-            test-path: "tests/integration/database/test_roi_observed.py"
-            seed: none
-            workers: 0
-            timeout-minutes: 10
-            job-timeout-minutes: 35
-
-          - shard: proxy-behavior
-            test-path: "tests/proxy_behavior"
-            seed: db-push
-            workers: 0
-            timeout-minutes: 25
-            job-timeout-minutes: 50
-
-          - shard: proxy-security
-            test-path: "tests/proxy_security_tests"
-            seed: db-push
-            workers: 0
-            timeout-minutes: 15
-            job-timeout-minutes: 40
-
-          - shard: schema-migration
-            test-path: "tests/proxy_migration_tests"
-            seed: none
-            workers: 0
-            timeout-minutes: 20
-            job-timeout-minutes: 45
-
-    env:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        timeout-minutes: 3
-        with:
-          persist-credentials: false
-
-      - name: Detect relevant changes
-        id: changes
-        timeout-minutes: 2
-        uses: ./.github/actions/detect-changes
-
-      - name: Set up Python
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 3
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 3
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
-        timeout-minutes: 5
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-postgres-
-
-      - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
-        timeout-minutes: 5
-        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-postgres-
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 12
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
-
-      - name: Cache Prisma binaries
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 3
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 5
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Seed database schema
-        if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
-        timeout-minutes: 10
-        run: |
-          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-
-      - name: Run tests
-        if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: ${{ matrix.timeout-minutes }}
-        env:
-          TEST_PATH: ${{ matrix.test-path }}
-          WORKERS: ${{ matrix.workers }}
-          PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || matrix.shard == 'roi-database' && '--cov=./litellm --cov-report=xml:coverage-roi-postgres.xml' || '' }}
-        run: |
-          if [ "${WORKERS}" = "0" ]; then
-            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
-          else
-            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
-          fi
-
-      - name: Upload Lens database coverage
-        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
-        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
-        with:
-          use_oidc: true
-          version: v11.3.1
-          root_dir: ${{ github.workspace }}
-          files: coverage-lens-postgres.xml
-          flags: lens-postgres
-          fail_ci_if_error: true
-
-      - name: Upload ROI database coverage
-        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'roi-database' && !cancelled()
-        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
-        with:
-          use_oidc: true
-          version: v11.3.1
-          root_dir: ${{ github.workspace }}
-          files: coverage-roi-postgres.xml
-          flags: roi-postgres
-          fail_ci_if_error: true

.github/workflows/test-redis-compat.yml

@@ -1,106 +0,0 @@
-name: "Unit Tests: Redis Client Version Compatibility"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "litellm_**"
-    paths:
-      - "litellm/_redis.py"
-      - "litellm/_redis_credential_provider.py"
-      - "litellm/caching/redis_cache.py"
-      - "litellm/caching/evicted_client_closer.py"
-      - "tests/unit/test_redis.py"
-      - "tests/local_testing/test_caching.py"
-      - "tests/unit/caching/test_redis_connection_pool.py"
-      - "tests/unit/caching/test_redis_cluster_cache.py"
-      - "tests/unit/caching/test_evicted_client_closer.py"
-      - ".github/workflows/test-redis-compat.yml"
-      - "pyproject.toml"
-      - "uv.lock"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  redis-compat:
-    name: "redis-py ${{ matrix.redis-version }}"
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    permissions:
-      contents: read
-      id-token: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
-        # newer legs prove the inspect.signature introspection in litellm/_redis.py
-        # keeps extracting kwargs on the redis-py releases people actually run now.
-        # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
-        # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
-        # for the 6.x line.
-        redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Install dependencies
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router
-
-      - name: Pin redis-py to the matrix version
-        env:
-          REDIS_VERSION: ${{ matrix.redis-version }}
-        run: |
-          uv pip install "redis==${REDIS_VERSION:?}"
-          uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
-
-      - name: Build Redis for cluster authentication tests
-        run: |
-          curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz"
-          echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e  $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP"
-          make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server
-          echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH"
-
-      - name: Run redis unit tests
-        run: |
-          redis-server --version
-          uv run --no-sync pytest \
-            tests/unit/test_redis.py \
-            tests/unit/caching/test_redis_connection_pool.py \
-            tests/unit/caching/test_redis_cluster_cache.py \
-            tests/unit/caching/test_evicted_client_closer.py \
-            tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
-            tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
-            --tb=short -vv \
-            --reruns 2 \
-            --reruns-delay 1 \
-            --durations=20 \
-            --cov=./litellm --cov-report=xml:coverage-redis.xml
-
-      - name: Upload Redis coverage
-        if: matrix.redis-version == '5.3.1'
-        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
-        with:
-          use_oidc: true
-          files: coverage-redis.xml
-          flags: redis-compat
-          fail_ci_if_error: false

.github/workflows/test-unit-proxy-db.yml

@@ -20,13 +20,6 @@ concurrency:
 # rather than alphabetical letter ranges. Adding a new test file means adding it
 # to whichever group it belongs to, not reshuffling slices.
 #
-# `.circleci/tests.yml` runs each group's files on same-repo events under the
-# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
-# the file lists. That pipeline is manual-only while the tests migrate, so
-# `unit-flag` makes the shard run that list on every event. `test-path` keeps
-# the files that still reach real providers and never left
-# tests/proxy_unit_tests.
-#
 # Design targets:
 #   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
@@ -78,93 +71,136 @@ jobs:
         include:
           # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
-            test-path: ""
-            unit-flag: proxy-db-key-generation
+            test-path: >-
+              tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
             workers: 0
             dist: loadscope
             timeout: 20
 
           # ---- auth: split into 2 shards ----
           - test-group: auth-checks
-            test-path: ""
-            unit-flag: proxy-db-auth-checks
+            test-path: >-
+              tests/unit/proxy/auth/test_auth_checks.py
+              tests/unit/proxy/auth/test_user_api_key_auth.py
+              tests/unit/proxy/test_credential_slot_registry.py
+              tests/unit/proxy/test_deprecated_key_grace_period.py
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: jwt-and-keys
-            test-path: ""
-            unit-flag: proxy-db-jwt-and-keys
+            test-path: >-
+              tests/unit/proxy/auth/test_jwt.py
+              tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
+              tests/unit/proxy/test_proxy_custom_auth.py
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- test_proxy_utils.py, single shard, worksteal distribution ----
           - test-group: proxy-utils
-            test-path: ""
-            unit-flag: proxy-db-proxy-utils
+            test-path: >-
+              tests/unit/proxy/test_proxy_utils.py
             workers: 4
             dist: worksteal
             timeout: 15
 
           # ---- proxy server: split into 2 shards ----
           - test-group: proxy-server-core
-            test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
-            unit-flag: proxy-db-proxy-server-core
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
+              tests/unit/proxy/test__lazy_features.py
+              tests/unit/proxy/test_aproxy_startup.py
+              tests/unit/proxy/test_proxy_server.py
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: proxy-runtime
-            test-path: ""
-            unit-flag: proxy-db-proxy-runtime
+            test-path: >-
+              tests/unit/proxy/auth/test_multipart_bypass_repro.py
+              tests/unit/proxy/auth/test_proxy_routes.py
+              tests/unit/proxy/middleware/test_request_size_limit_middleware.py
+              tests/unit/proxy/test_proxy_config_unit_test.py
+              tests/unit/proxy/test_proxy_token_counter.py
+              tests/unit/proxy/test_server_root_path.py
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- logging: split into 2 shards ----
           - test-group: custom-logging
-            test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
-            unit-flag: proxy-db-custom-logging
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_custom_logger.py
+              tests/unit/proxy/test_custom_callback_input.py
+              tests/unit/proxy/test_custom_logger_s3_gcs.py
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: logging-misc
-            test-path: ""
-            unit-flag: proxy-db-logging-misc
+            test-path: >-
+              tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
+              tests/unit/proxy/spend_tracking/test_search_api_logging.py
+              tests/unit/proxy/test_proxy_reject_logging.py
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: db-and-spend
-            test-path: ""
-            unit-flag: proxy-db-db-and-spend
+            test-path: >-
+              tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
+              tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
+              tests/unit/proxy/db/test_update_daily_tag_spend.py
+              tests/unit/proxy/test_db_schema_changes.py
+              tests/unit/proxy/test_prisma_client_backoff_retry.py
+              tests/unit/proxy/test_update_spend.py
+              tests/unit/skills/test_skills_db.py
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- guardrails + budget + hooks: split into 2 ----
           - test-group: guardrails-hooks
-            test-path: ""
-            unit-flag: proxy-db-guardrails-hooks
+            test-path: >-
+              tests/unit/proxy/hooks/test_banned_keyword_list.py
+              tests/unit/proxy/test_proxy_setting_guardrails.py
+              tests/unit/proxy/test_unit_test_proxy_hooks.py
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: budgets
-            test-path: ""
-            unit-flag: proxy-db-budgets
+            test-path: >-
+              tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+              tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
+              tests/unit/proxy/test_zero_cost_model_budget_bypass.py
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: endpoints-and-responses
-            test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
-            unit-flag: proxy-db-endpoints-and-responses
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_exception_mapping.py
+              tests/unit/proxy/lens
+              tests/unit/proxy/auth/test_models_fallback_endpoint.py
+              tests/unit/proxy/common_utils/test_check_batch_cost.py
+              tests/unit/proxy/common_utils/test_check_responses_cost.py
+              tests/unit/proxy/common_utils/test_realtime_cache.py
+              tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
+              tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
+              tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
+              tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
+              tests/unit/proxy/response_polling
+              tests/unit/proxy/test_custom_tokenizer_bug.py
+              tests/unit/proxy/test_get_favicon.py
+              tests/unit/proxy/test_get_image.py
+              tests/unit/proxy/test_prompt_test_endpoint.py
+              tests/unit/proxy/test_reducto_ocr_route.py
+              tests/unit/proxy/test_response_polling_pre_call_checks.py
+              tests/unit/proxy/test_ui_path_detection.py
             workers: 4
             dist: loadscope
             timeout: 15
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}
-      unit-flag: ${{ matrix.unit-flag }}
       workers: ${{ matrix.workers }}
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}

.github/workflows/test-unit.yml

@@ -21,10 +21,8 @@ concurrency:
 # files that each wrapped a single call to _test-unit-base.yml. Adding a shard is
 # now one matrix entry rather than a new file.
 #
-# `name` is the shard id and nothing else, so each check reports as
-# "<shard> / Run tests" exactly as it did when the shard had its own file. Those
-# strings are the branch ruleset's required contexts, so they are load-bearing:
-# renaming an entry renames a required check and the ruleset stops matching it.
+# `name` is the shard id, and each check reports as "<shard> / Run tests".
+# Unit shard names are not required ruleset contexts, so matrix entries can be split freely.
 #
 # Every entry states its timeouts even when they equal the base workflow's
 # defaults. An absent matrix key renders as an empty string, which is not a
@@ -35,13 +33,102 @@ concurrency:
 # a matrix and carries a shard-coverage guard that reads that file by name.
 # Folding it in here is a follow-up, together with generalising that guard into
 # assert_ci_coverage.py.
-#
-# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the
-# shard under the same Codecov flag. That pipeline is manual-only while the
-# tests migrate, so the shard also runs those files on every event.
 jobs:
+  rust-bridge:
+    name: Build the Rust bridge
+    outputs:
+      artifact: ${{ steps.rust-bridge-artifact.outputs.name }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    env:
+      UV_PYTHON: "3.12"
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Define editable Rust bridge cache key
+        id: rust-bridge-key
+        if: steps.changes.outputs.decision != 'skip'
+        env:
+          RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
+        run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Restore editable Rust bridge
+        id: rust-bridge-cache
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
+        with:
+          path: rust-bridge-dist
+          key: ${{ steps.rust-bridge-key.outputs.key }}
+
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Build editable Rust bridge
+        if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
+        timeout-minutes: 15
+        run: |
+          mkdir -p rust-bridge-dist
+          uv run --no-project --with maturin==1.15.0 python -c 'import maturin; maturin.build_editable("rust-bridge-dist")'
+          strip --strip-debug litellm/rust_bridge/_native.abi3.so
+          mkdir -p rust-bridge-dist/litellm/rust_bridge
+          cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
+
+      - name: Upload Rust bridge artifact
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 10
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1
+        with:
+          name: rust-bridge-${{ github.run_id }}-${{ github.run_attempt }}
+          path: rust-bridge-dist/
+          retention-days: 1
+
+      - name: Save editable Rust bridge
+        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
+        timeout-minutes: 10
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
+        with:
+          path: rust-bridge-dist
+          key: ${{ steps.rust-bridge-key.outputs.key }}
+
+      - name: Expose the Rust bridge artifact
+        id: rust-bridge-artifact
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 1
+        env:
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        run: echo "name=rust-bridge-${RUN_ID}-${RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+
   unit:
     name: ${{ matrix.shard }}
+    needs: rust-bridge
+    if: ${{ !cancelled() }}
     permissions:
       contents: read
       id-token: write
@@ -50,29 +137,35 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - shard: mcp-integration
-            artifact-name: mcp-integration
-            test-path: "tests/mcp_tests"
-            unit-flag: mcp-integration
-            workers: 2
-            reruns: 0
-            timeout-minutes: 20
-            job-timeout-minutes: 60
-
           - shard: core-utils
             artifact-name: core-utils
-            test-path: tests/unit/decisions
-            unit-flag: core-utils
-            workers: 2
-            reruns: 1
+            test-path: >-
+              tests/unit/decisions
+              tests/unit/litellm_core_utils
+            workers: 4
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
-            test-path: ""
-            unit-flag: enterprise-routing
-            workers: 2
+            test-path: >-
+              tests/unit/google_genai
+              tests/unit/router_strategy
+              tests/unit/router_utils
+              tests/unit/proxy/common_utils/test_cache_aware_routing.py
+              tests/unit/enterprise/enterprise_callbacks/send_emails
+              tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
+              tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
+              tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
+              tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
+              tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+              tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
+              tests/unit/enterprise/proxy/test_enterprise_routes.py
+              tests/unit/enterprise/proxy/test_file_deletion_blocking.py
+              tests/unit/enterprise/proxy/test_managed_files_access_check.py
+              tests/unit/enterprise/proxy/test_managed_files_hook.py
+            workers: 4
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
@@ -82,27 +175,41 @@ jobs:
             test-path: >-
               tests/test_litellm/integrations
               tests/test_litellm/tracing
-            unit-flag: integrations
-            workers: 2
-            reruns: 3
+              tests/unit/integrations
+            workers: 4
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
-            test-path: ""
-            unit-flag: llm-vertex-ai
-            workers: 1
-            reruns: 2
+            test-path: >-
+              tests/unit/llms/vertex_ai
+            workers: 4
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
-            test-path: ""
-            unit-flag: llm-other-providers
-            workers: 2
-            reruns: 2
+            test-path: >-
+              tests/unit/llms
+              --ignore=tests/unit/llms/vertex_ai
+              --ignore=tests/unit/llms/openai
+              --ignore=tests/unit/llms/meta
+              --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: OpenAI and Meta Providers
+            artifact-name: llm-openai-meta
+            test-path: >-
+              tests/unit/llms/openai
+              tests/unit/llms/meta
+            workers: 4
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
@@ -110,19 +217,44 @@ jobs:
             artifact-name: misc
             test-path: >-
               tests/test_litellm/test_*.py
-            unit-flag: misc
-            workers: 2
+              tests/unit/test_*.py
+            workers: 4
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
 
+          - shard: misc-dirs
+            artifact-name: misc-dirs
+            test-path: >-
+              tests/unit/test_router
+              tests/unit/a2a_protocol
+              tests/unit/batches
+              tests/unit/chat_completions
+              tests/unit/completion_extras
+              tests/unit/containers
+              tests/unit/embeddings
+              tests/unit/endpoints
+              tests/unit/files
+              tests/unit/harness
+              tests/unit/images
+              tests/unit/interactions
+              tests/unit/messages
+              tests/unit/rag
+              tests/unit/rerank_api
+              tests/unit/rust_bridge
+              tests/unit/secret_managers
+              tests/unit/vector_stores
+              tests/unit/videos
+              --ignore=tests/unit/rust_bridge/native_route_wheel_test.py
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
           - shard: proxy-auth
             artifact-name: proxy-auth
             test-path: >-
               tests/unit/proxy/auth
-              tests/unit/proxy/hooks
-              tests/unit/proxy/policy_engine
-              tests/unit/proxy/client
               --ignore=tests/unit/proxy/auth/test_auth_checks.py
               --ignore=tests/unit/proxy/auth/test_user_api_key_auth.py
               --ignore=tests/unit/proxy/auth/test_default_end_user_budget_simple.py
@@ -130,35 +262,56 @@ jobs:
               --ignore=tests/unit/proxy/auth/test_models_fallback_endpoint.py
               --ignore=tests/unit/proxy/auth/test_multipart_bypass_repro.py
               --ignore=tests/unit/proxy/auth/test_proxy_routes.py
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: proxy-hooks-client
+            artifact-name: proxy-hooks-client
+            test-path: >-
+              tests/unit/proxy/hooks
+              tests/unit/proxy/policy_engine
+              tests/unit/proxy/client
               --ignore=tests/unit/proxy/hooks/test_banned_keyword_list.py
               --ignore=tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
-            workers: 2
-            reruns: 2
+            workers: 4
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
             test-path: >-
+              tests/unit/proxy/management_endpoints
+              tests/unit/proxy/management_helpers
+              tests/unit/proxy/list_api
               tests/unit/proxy/analytics_endpoints
               tests/unit/proxy/decisions_endpoints
-              tests/unit/proxy/management_endpoints
-              tests/unit/proxy/list_api
               tests/unit/proxy/memory
-              tests/unit/proxy/guardrails
-              tests/unit/proxy/management_helpers
+              tests/unit/proxy/agent_endpoints
+              tests/unit/proxy/openai_files_endpoint
+              tests/unit/proxy/health_endpoints
+              tests/unit/proxy/batches_endpoints
               --ignore=tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
               --ignore=tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
               --ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
               --ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: proxy-feature-endpoints
+            artifact-name: proxy-feature-endpoints
+            test-path: >-
+              tests/unit/proxy/guardrails
               --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
               --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
               --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
               --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
               tests/unit/proxy/anthropic_endpoints
               tests/unit/proxy/google_endpoints
-              tests/unit/proxy/openai_files_endpoint
-              tests/unit/proxy/batches_endpoints
               tests/unit/proxy/container_endpoints
               tests/unit/proxy/fine_tuning_endpoints
               tests/unit/proxy/vector_store_files_endpoints
@@ -168,11 +321,9 @@ jobs:
               tests/unit/proxy/ocr_endpoints
               tests/unit/proxy/search_endpoints
               tests/unit/proxy/vector_store_endpoints
-              tests/unit/proxy/agent_endpoints
               tests/unit/proxy/a2a
               tests/unit/proxy/credential_endpoints
               tests/unit/proxy/discovery_endpoints
-              tests/unit/proxy/health_endpoints
               tests/unit/proxy/shutdown
               tests/unit/proxy/public_endpoints
               tests/unit/proxy/prompts
@@ -183,7 +334,7 @@ jobs:
               tests/unit/proxy/config_resolvers
               tests/unit/proxy/utils
             workers: 4
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
@@ -191,7 +342,7 @@ jobs:
             artifact-name: proxy-server
             test-path: "tests/unit/proxy/proxy_server"
             workers: 4
-            reruns: 2
+            reruns: 0
             timeout-minutes: 60
             job-timeout-minutes: 100
 
@@ -218,9 +369,12 @@ jobs:
               tests/unit/proxy/enterprise_billing
               tests/unit/proxy/types_utils
               tests/unit/proxy/logging_endpoints
-            unit-flag: proxy-infra
+              tests/unit/gateway
+              tests/unit/proxy/management
+              tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+              tests/unit/proxy/roi_calculator
             workers: 4
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
@@ -254,52 +408,92 @@ jobs:
               --ignore=tests/unit/proxy/test_update_spend.py
               --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py
             workers: 4
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: caching-local
             artifact-name: caching-local
-            test-path: ""
-            unit-flag: caching-local
+            test-path: >-
+              tests/unit/caching
             workers: 2
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: proxy-extras
             artifact-name: proxy-extras
-            test-path: ""
-            unit-flag: proxy-extras
+            test-path: >-
+              tests/unit/litellm_proxy_extras
             workers: 2
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
           - shard: enterprise-package
             artifact-name: enterprise-package
-            test-path: ""
-            unit-flag: enterprise-package
+            test-path: >-
+              tests/unit/enterprise/integrations
+              tests/unit/enterprise/proxy/auth
+              tests/unit/enterprise/proxy/guardrails
+              tests/unit/enterprise/proxy/management_endpoints
+              tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
+              tests/unit/enterprise/proxy/test_liteadmin.py
+              tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
             workers: 4
-            reruns: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
 
+          - shard: enterprise-managed-files
+            artifact-name: enterprise-managed-files
+            test-path: >-
+              tests/unit/enterprise/proxy/hooks
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+            dist: load
+
           - shard: responses-caching-types
             artifact-name: responses-caching-types
-            test-path: ""
-            unit-flag: responses-caching-types
+            test-path: >-
+              tests/unit/responses
+              tests/unit/types
+              --ignore=tests/unit/responses/mcp
             workers: 2
-            reruns: 2
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: unit
+            artifact-name: unit
+            test-path: >-
+              tests/unit/anthropic_interface
+              tests/unit/compression
+              tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py
+              tests/unit/enterprise/enterprise_callbacks/test_llm_guard.py
+              tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py
+              tests/unit/integration_support
+              tests/unit/models
+              tests/unit/ocr
+              tests/unit/passthrough
+              tests/unit/realtime_api
+              tests/unit/repositories
+              tests/unit/sandbox
+              tests/unit/skills/test_skills_main.py
+              tests/unit/tracing
+            workers: 2
+            reruns: 0
             timeout-minutes: 20
             job-timeout-minutes: 60
     uses: ./.github/workflows/_test-unit-base.yml
     with:
+      rust-bridge-artifact: ${{ needs.rust-bridge.outputs.artifact }}
       test-path: ${{ matrix.test-path }}
-      unit-flag: ${{ matrix.unit-flag || '' }}
       workers: ${{ matrix.workers }}
       reruns: ${{ matrix.reruns }}
       timeout-minutes: ${{ matrix.timeout-minutes }}
       job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
+      dist: ${{ matrix.dist || 'loadscope' }}
       artifact-name: ${{ matrix.artifact-name }}
-      legacy-mcp-peer: ${{ matrix.shard == 'mcp-integration' }}

Dockerfile

@@ -79,6 +79,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
 RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
+    --group admin-mcp \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
@@ -101,6 +102,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.
 RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
+    --group admin-mcp \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \

Makefile

@@ -58,7 +58,7 @@ help:
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
 	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
-	@echo "  make lens-dev           - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)"
+	@echo "  make lens-dev           - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large --seed-logs\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)"
 	@echo ""
 	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
 	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."

backend/Dockerfile

@@ -44,6 +44,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
+        --group admin-mcp \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
@@ -56,6 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
+        --group admin-mcp \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \

backend/routes/allowlist.py

@@ -85,6 +85,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/analytics/",
     "/lens/",
     "/v1/traces",
+    "/v1/logs",
     "/global/",
     "/user_agent",
     "/usage/",
@@ -160,6 +161,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
 
 BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
     {
+        "/admin",
         "/swagger",  # API documentation static assets belong to the backend
         "/mcp",  # lazily-mounted MCP sub-app serves on the backend component
     }

ci_cd/TEST_KEY_PATTERNS.md

@@ -5,7 +5,7 @@ Standard patterns for test/mock keys and credentials in the LiteLLM codebase to
 ## How GitGuardian Works
 
 GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
-- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
+- **Low entropy** values (like `sk-test`, `postgres`) are automatically ignored
 - **High entropy** values (realistic-looking secrets) trigger detection
 - **Context-aware** detection understands code syntax like `os.environ["KEY"]`
 
@@ -15,8 +15,8 @@ GitGuardian uses **machine learning and entropy analysis**, not just pattern mat
 These won't trigger GitGuardian's ML detector:
 
 ```python
-api_key = "sk-1234"
-api_key = "sk-12345"
+api_key = "sk-test"
+api_key = "sk-mock"
 database_password = "postgres"
 token = "test123"
 ```

cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb

@@ -16,6 +16,8 @@
         "\n",
         "These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n",
         "\n",
+        "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running authenticated examples\n",
+        "\n",
         "For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n",
         "\n",
         "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
@@ -95,9 +97,10 @@
       },
       "outputs": [],
       "source": [
+        "import os\n",
         "from openai import OpenAI\n",
         "client = OpenAI(\n",
-        "    api_key=\"sk-1234\", # [OPTIONAL] set if you set one on proxy, else set \"\"\n",
+        "    api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
         "    base_url=\"http://0.0.0.0:4000\",\n",
         ")\n",
         "\n",
@@ -298,14 +301,14 @@
         "    engine=\"azure-gpt-3.5\",               # model_name on litellm proxy\n",
         "    temperature=0.0,\n",
         "    azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n",
-        "    api_key=\"sk-1234\",                    # litellm proxy API Key\n",
+        "    api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
         "    api_version=\"2023-07-01-preview\",\n",
         ")\n",
         "\n",
         "embed_model = AzureOpenAIEmbedding(\n",
         "    deployment_name=\"azure-embedding-model\",\n",
         "    azure_endpoint=\"http://0.0.0.0:4000\",\n",
-        "    api_key=\"sk-1234\",\n",
+        "    api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
         "    api_version=\"2023-07-01-preview\",\n",
         ")\n",
         "\n",
@@ -341,7 +344,7 @@
         "\n",
         "const model = new ChatOpenAI({\n",
         "  modelName: \"gpt-4\",\n",
-        "  openAIApiKey: \"sk-1234\",\n",
+        "  openAIApiKey: process.env.LITELLM_MASTER_KEY,\n",
         "  modelKwargs: {\"metadata\": \"hello world\"} // 👈 PASS Additional params here\n",
         "}, {\n",
         "  basePath: \"http://0.0.0.0:4000\",\n",
@@ -372,7 +375,7 @@
         "const { OpenAI } = require('openai');\n",
         "\n",
         "const openai = new OpenAI({\n",
-        "  apiKey: \"sk-1234\", // This is the default and can be omitted\n",
+        "  apiKey: process.env.LITELLM_MASTER_KEY,\n",
         "  baseURL: \"http://0.0.0.0:4000\"\n",
         "});\n",
         "\n",

cookbook/Proxy_Batch_Users.ipynb

@@ -6,7 +6,9 @@
     "id": "680oRk1af-xJ"
    },
    "source": [
-    "# Environment Setup"
+    "# Environment Setup\n",
+    "\n",
+    "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running the cells"
    ]
   },
   {
@@ -17,6 +19,7 @@
    },
    "outputs": [],
    "source": [
+    "import os\n",
     "import csv\n",
     "from typing import Optional\n",
     "import httpx\n",
@@ -24,7 +27,7 @@
     "import asyncio\n",
     "\n",
     "proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n",
-    "master_key = \"sk-1234\" # 👈 SET TO PROXY MASTER KEY"
+    "master_key = os.environ[\"LITELLM_MASTER_KEY\"]"
    ]
   },
   {

cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md

@@ -47,7 +47,7 @@ Set your environment variables:
 
 ```bash
 export ANTHROPIC_API_KEY="your-anthropic-api-key"
-export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
+export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
 ```
 
 ## Step 2: Start Proxy
@@ -292,4 +292,3 @@ model_list:
 - [LiteLLM Documentation](https://docs.litellm.ai/)
 - [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
 - [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
-

cookbook/anthropic_agent_sdk/README.md

@@ -20,6 +20,12 @@ litellm --model claude-sonnet-4-20250514
 litellm --config config.yaml
 ```
 
+Generate a master key before starting the proxy:
+
+```bash
+export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
+```
+
 ### 3. Run the chat
 
 **Basic Agent (no MCP):**
@@ -58,7 +64,7 @@ Set these environment variables if needed:
 
 ```bash
 export LITELLM_PROXY_URL="http://localhost:4000"
-export LITELLM_API_KEY="sk-1234"
+export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
 export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
 ```
 
@@ -98,7 +104,7 @@ The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
 ```python
 # Point to LiteLLM gateway (not Anthropic)
 os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
-os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM key
+os.environ["ANTHROPIC_API_KEY"] = os.environ["LITELLM_API_KEY"]
 
 # Use any model configured in LiteLLM
 options = ClaudeAgentOptions(

cookbook/anthropic_agent_sdk/common.py

@@ -13,7 +13,7 @@ class Config:
     LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
 
     # LiteLLM API key (master key or virtual key)
-    LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+    LITELLM_API_KEY = os.environ["LITELLM_API_KEY"]
 
     # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
     LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -1,8 +1,10 @@
+import os
+
 from openai import OpenAI
 
 client = OpenAI(
     base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
+    api_key=os.environ["LITELLM_MASTER_KEY"],
 )
 
 BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -4,10 +4,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
 When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
 """
 
+import os
+
 import openai
 
 client = openai.OpenAI(
-    api_key="sk-1234",  # paste your litellm proxy api key here
+    api_key=os.environ["LITELLM_MASTER_KEY"],
     base_url="http://localhost:4000",  # paste your litellm proxy base url here
 )
 print("Making API request to Responses API with MCP tools")

cookbook/livekit_agent_sdk/README.md

@@ -15,12 +15,14 @@ pip install livekit-agents[xai] websockets
 ```bash
 # With xAI
 export XAI_API_KEY="your-xai-key"
+export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
 litellm --config config.yaml --port 4000
 ```
 
 ### 3. Run the voice agent
 
 ```bash
+export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
 python main.py
 ```
 
@@ -32,7 +34,7 @@ Set these environment variables if needed:
 
 ```bash
 export LITELLM_PROXY_URL="http://localhost:4000"
-export LITELLM_API_KEY="sk-1234"
+export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
 export LITELLM_MODEL="grok-voice-agent"
 ```
 
@@ -59,7 +61,7 @@ model_list:
       mode: realtime
 
 general_settings:
-  master_key: sk-1234
+  master_key: os.environ/LITELLM_MASTER_KEY
 ```
 
 Then start: `litellm --config config.yaml --port 4000`
@@ -73,7 +75,7 @@ from livekit.plugins import xai
 
 model = xai.realtime.RealtimeModel(
     voice="ara",
-    api_key="sk-1234",              # LiteLLM proxy key
+    api_key=os.environ["LITELLM_MASTER_KEY"],
     base_url="http://localhost:4000", # Point to LiteLLM
 )
 ```

cookbook/livekit_agent_sdk/config.example.yaml

@@ -17,4 +17,4 @@ litellm_settings:
   drop_params: True
 
 general_settings:
-  master_key: sk-1234  # Change this to a secure key
+  master_key: os.environ/LITELLM_MASTER_KEY

cookbook/livekit_agent_sdk/main.py

@@ -13,7 +13,7 @@ import websockets
 
 # Configuration
 PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
-API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+API_KEY = os.environ["LITELLM_API_KEY"]
 MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
 
 

cookbook/misc/config.yaml

@@ -58,7 +58,7 @@ litellm_settings:
   context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
 
 general_settings: 
-  master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys
+  master_key: os.environ/LITELLM_MASTER_KEY
   store_model_in_db: True
   proxy_budget_rescheduler_min_time: 60
   proxy_budget_rescheduler_max_time: 64

cookbook/misc/migrate_proxy_config.py

@@ -12,6 +12,8 @@ Step 2: reads `model_list` and loops through all models
 Step 3: calls `<proxy-base-url>/model/new` for each model
 """
 
+import os
+
 import yaml
 import requests
 
@@ -89,7 +91,7 @@ def migrate_models(config_file, proxy_base_url):
 # Usage
 config_file = "config.yaml"
 proxy_base_url = "http://0.0.0.0:4000"
-master_key = "sk-1234"
+master_key = os.environ["LITELLM_MASTER_KEY"]
 print(f"config_file: {config_file}")
 print(f"proxy_base_url: {proxy_base_url}")
 migrate_models(config_file, proxy_base_url)

cookbook/misc/test_responses_api.py

@@ -1,8 +1,10 @@
 import base64
+import os
+
 from openai import OpenAI
 import time
 
-client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
+client = OpenAI(base_url="http://0.0.0.0:4001", api_key=os.environ["LITELLM_MASTER_KEY"])
 
 
 # Function to encode the image

cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb

@@ -4,7 +4,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n"
+    "# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n",
+    "\n",
+    "Set `LITELLM_PROXY_API_KEY` in the notebook environment to a key accepted by the proxy before running the model cell\n"
    ]
   },
   {
@@ -160,11 +162,13 @@
    },
    "outputs": [],
    "source": [
+    "import os\n",
+    "\n",
     "model = ChatOpenAI(\n",
     "    openai_api_base=\"LITELLM_PROXY_BASE_URL\", # e.g.: http://0.0.0.0:4000\n",
     "    model = \"gpt-3.5-turbo\", # LITELLM 'model_name'\n",
     "    temperature=0.1, \n",
-    "    api_key=\"LITELLM_PROXY_API_KEY\" # e.g.: \"sk-1234\"\n",
+    "    api_key=os.environ[\"LITELLM_PROXY_API_KEY\"]\n",
     ")"
    ]
   },

cookbook/mock_prompt_management_server/README.md

@@ -69,7 +69,7 @@ litellm --config config.yaml
 ```bash
 curl http://0.0.0.0:4000/v1/chat/completions \
   -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
   -d '{
     "model": "gpt-3.5-turbo",
     "prompt_id": "hello-world-prompt",
@@ -290,4 +290,3 @@ Before deploying to production:
 ## Questions?
 
 This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
-

cookbook/nova_sonic_realtime.py

@@ -10,6 +10,7 @@ Prerequisites:
 - websockets installed: pip install websockets
 
 Usage:
+    export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
     python nova_sonic_realtime.py
 """
 
@@ -33,7 +34,7 @@ CHUNK_SIZE = 1024
 
 # LiteLLM proxy configuration
 LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
-LITELLM_API_KEY = "sk-12345"  # Your LiteLLM API key
+LITELLM_API_KEY = os.environ["LITELLM_MASTER_KEY"]
 
 
 class RealtimeClient:

cookbook/veo_video_generation.py

@@ -24,8 +24,8 @@ class VeoVideoGenerator:
 
     def __init__(
         self,
+        api_key: str,
         base_url: str = "http://localhost:4000/gemini/v1beta",
-        api_key: str = "sk-1234",
     ):
         """
         Initialize the Veo video generator.
@@ -274,12 +274,12 @@ def main():
 
     Configure these environment variables:
     - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
-    - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
+    - LITELLM_API_KEY: API key for LiteLLM proxy authentication
     """
 
     # Configuration from environment or defaults
     base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
-    api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+    api_key = os.environ["LITELLM_API_KEY"]
 
     print("🚀 Starting Veo Video Generation Example")
     print(f"📡 Using LiteLLM proxy at: {base_url}")

deploy/lens/Dockerfile

@@ -6,23 +6,30 @@ FROM $UV_IMAGE AS uvbin
 
 FROM $LITELLM_BUILD_IMAGE AS builder
 COPY --from=uvbin /uv /usr/local/bin/uv
-RUN apk add --no-cache python-3.13
+RUN apk add --no-cache python-3.13 build-base libseccomp-dev
 ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy
 WORKDIR /app
 COPY deploy/lens/requirements.lock /tmp/requirements.lock
 RUN uv venv --python python3.13 /app/.venv && \
     uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock
+COPY deploy/lens/python_policy.c /tmp/python_policy.c
+RUN cc -std=c11 -D_GNU_SOURCE -O2 -Wall -Wextra -Werror /tmp/python_policy.c -lseccomp -o /tmp/python-policy && \
+    /tmp/python-policy /app/python.seccomp
 
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 ARG LITELLM_RELEASE_TAG=""
 RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
-RUN apk add --no-cache python-3.13
+RUN apk add --no-cache python-3.13 setpriv
 ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONDONTWRITEBYTECODE=1
 WORKDIR /app
 COPY --from=builder /app/.venv /app/.venv
 COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
+COPY litellm/proxy/lens/context_pipeline.py litellm/proxy/lens/agent_review.py litellm/proxy/lens/agent_runtime.py litellm/proxy/lens/agent_workspace.py litellm/proxy/lens/python_tool.py litellm/proxy/lens/activity.py litellm/proxy/lens/agent_context.py /app/lens/
 COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
+COPY --from=builder /app/python.seccomp /app/lens/python.seccomp
+COPY deploy/lens/python_runtime.py /tmp/python_runtime.py
+RUN python3.13 -S /tmp/python_runtime.py /app/lens/python-runtime.json && rm /tmp/python_runtime.py
 USER 65532:65532
 CMD ["python", "-m", "lens.worker"]

deploy/lens/README.md

@@ -21,7 +21,7 @@ docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
 docker compose -f docker/docker-compose.tracing.yml up -d --build
 ```
 
-Open `http://localhost:4002/ui/` and sign in as `admin` with password `sk-1234`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
+Open `http://localhost:4002/ui/` and sign in as `admin` with the key saved in `.lens-dev/master_key`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
 
 ```bash
 export LITELLM_URL=http://litellm:4000
@@ -100,9 +100,9 @@ docker compose --env-file /path/to/lens.env -f compose.yaml up -d
 
 To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
 
-The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
+The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. Python reports storage failures to the reviewer and cleans up temporary files, so the reviewer can retry a smaller computation or report insufficient evidence. The worker remains available for other scans. Existing workers must be recreated with the new image and mount options
 
-The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
+The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles up to three investigations concurrently and can serve multiple lenses. For more throughput, start another worker with a separate credential
 
 If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
 
@@ -116,7 +116,7 @@ Describe how the agent should behave and optionally add specific checks. Select
 
 Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
 
-Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
+Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
 
 ## Read the results
 
@@ -132,11 +132,13 @@ The proxy selects executions received or updated within the configured lookback
 
 A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
 
-The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence
+The worker prepares a workspace containing the selected execution metadata and reviews executions in parallel. Reviewers receive their assignment and use catalog, read, search and optional Python tools to inspect evidence, including nested agents and other sampled executions. Tools retrieve original content from the gateway when requested; the worker does not preload the sampled traces or inject them into each model request. Python receives selected evidence as streamed input. Completed reviews retain cited excerpts and metadata. Observation batches are grouped in parallel, reconciled, and investigated against the original evidence. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs
 
-There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools
+There is no fixed total run, span, candidate or investigation-turn cutoff. Agents can replace their active conversation with working notes. If a request exceeds the configured model's context window, the worker compacts the conversation automatically and resumes with references to its archived tool history. Original evidence remains accessible through the gateway while it is available and retained. Tool results and working notes remain accessible during the investigation; character ranges make even a single oversized result readable in pieces. A review reports an error if the task or its replacement notes cannot fit. Context windows, the configured budget, worker resources and recorded evidence still bound practical work. The investigator has no browsing, code-editing or production-action tools
 
-Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
+The live review drawer shows loading, trace review, parallel grouping, reconciliation and candidate investigation. It reports current model and tool operations, including context compaction, and retains tool-call counts on completed trace reviews. These counts describe attempted calls, not successful executions. This progress channel contains operation metadata, not Python code or tool output. Preliminary observations remain separate from final findings; the final finding format and evidence links are unchanged
+
+Each model response must match its JSON schema. A malformed response gets one repair attempt through the same budget controls. A session review that remains invalid or cannot fit marks that execution unassessable while other reviews continue. Broken evidence pagination or missing content pages return tool errors so the agent can inspect narrower spans or other evidence. Unreadable citations receive repair feedback. Verified excerpts remain available without fetching their source again. The affected source counts as partial, including failures discovered during later investigations, while the reviewer owns its assessment. Candidate investigation errors preserve completed findings. Source and analysis errors remain visible and mark the final scan as failed; transport errors, cancellation and budget exhaustion stop the scan. Both the worker and proxy validate quoted evidence against original content. Per-run issue assessments follow supporting citations, including evidence found by another run's reviewer; counterexamples do not mark a run affected. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
 
 Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
 
@@ -178,16 +180,15 @@ Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or
 
 The default is Next.js dev with no production build (`LENS_DEV_BUILD_UI=0`). Set `LENS_DEV_BUILD_UI=1` when you also want a fresh static dashboard at `http://localhost:4000/ui/`. Build output goes to `.lens-dev/logs/ui-build.log`; a failed build stops startup. Both modes keep the live dashboard on port 3000. Startup checks the live login route before seeding and fails with the UI log path if Next.js exits. `LENS_DEV_STARTUP_TIMEOUT_SECONDS` controls startup readiness retries (default 300; `LENS_DEV_READINESS_REQUEST_TIMEOUT_SECONDS` caps each HTTP probe, default 5)
 
-For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies, over one million spans and linked request logs. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. The default profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks. Large seeds use the same parser and compressed ClickHouse writer in batches of four copies, and write matching request logs to PostgreSQL. The first and last batches verify linked spend totals through the proxy
+For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies spread over the last 24 hours, about 860,000 spans with linked request logs, plus three long sessions of roughly 1,150, 9,200 and 92,000 spans in a single trace for drawer paging and the oversized read path. Their trace IDs are printed at the end. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. Every profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks, and verifies linked spend totals through the proxy. Large seeds then copy that first copy inside ClickHouse and PostgreSQL with `INSERT ... SELECT`, rewriting trace, span and call IDs so each copy keeps its own spend, and verify the last copy through the proxy
 
-Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies, and `LENS_DEV_SEED_BATCH_COPIES` overrides copies per bulk insert (default 4, about 2,000 spans). Start with four or fewer on a constrained machine. Larger batches still respect the existing ClickHouse insert size limit; each capture is decoded separately within the OTLP safety budget. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
+Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
 
 Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse variables before starting the proxy and seeder so both processes use the same settings. Invalid, zero and negative values fail instead of silently falling back. Changing these limits does not require rebuilding Rust
 
 | Environment variable | Default | Controls |
 | --- | --- | --- |
 | `LENS_DEV_SEED_COPIES` | 1 default, 2000 large | Total fixture copies |
-| `LENS_DEV_SEED_BATCH_COPIES` | 4 | Copies per bulk insert |
 | `LENS_DEV_SEED_TIMEOUT_SECONDS` | 120 | Seeder HTTP timeout |
 | `OTLP_MAX_BODY_BYTES` | 16777216 | HTTP body and decompressed payload bytes |
 | `OTLP_MAX_CONCURRENT_INGESTS` | 2 | Concurrent proxy ingestion requests |
@@ -202,7 +203,7 @@ Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse va
 | `CLICKHOUSE_TRACE_MAX_INSERT_BYTES` | 67108864 | Encoded trace or spend insert bytes |
 | `CLICKHOUSE_INSERT_TIMEOUT_SECONDS` | 30 | ClickHouse insert HTTP timeout |
 
-The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits. Bulk seeding parses each capture separately, keeping the per-export limits distinct from the bulk insert limit. Use smaller batches if an insert exceeds its byte budget. For example, `LENS_DEV_SEED_COPIES=100 LENS_DEV_SEED_BATCH_COPIES=2 make lens-dev ARGS="--seed large"`
+The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits.
 
 ## Quality evaluation
 
@@ -216,7 +217,7 @@ python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
 
 Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
 
-The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
+The default workspace retrieves trace content on demand. Python calls have temporary scratch space that is removed after execution. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
 
 To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
 
@@ -243,3 +244,42 @@ The hourly development pipeline pins all component images to the same selected c
 ## Worker dependencies
 
 The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures
+
+## Python analysis boundary
+
+The `python` tool runs ordinary CPython with the standard library in a fresh child process inside the existing worker container. It receives the selected evidence as `data` over stdin and has its own temporary working directory. It creates no additional container or service. Read and search tools remain available independently of Python
+
+The native worker image builds a syscall policy with libseccomp and includes the full `setpriv` launcher. Each child starts with no inherited worker secrets or open worker files, isolated Python startup, Landlock filesystem restrictions and a default-deny seccomp filter. It can read the Python runtime and its own scratch files. Worker source, installed worker packages, other jobs' files and `/proc` contents are unavailable. Network sockets, child processes, cross-process memory operations, signals to other processes and filesystem metadata mutation are denied, including calls made through `ctypes`. Some metadata inspection, such as `stat`, `access` and `readlink` of known paths, remains possible
+
+Python execution requires a native Linux worker with Landlock ABI 3 or later and seccomp filtering. Build the image for the host architecture. Missing policy files, an incompatible kernel, or an unsupported host such as a macOS source worker returns a clear tool error. There is no unrestricted execution fallback. Keep the container's non-root user, dropped capabilities, no-new-privileges setting, read-only root and writable temporary mount
+
+The worker permits two Python children at once across all investigations. Set `LENS_PYTHON_CONCURRENCY` to a positive integer to change this worker-wide pool. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
+
+| Per-call resource | Default |
+| --- | --- |
+| Elapsed execution time | 60 seconds |
+| CPU time | 30 seconds |
+| Process address space | 512 MiB |
+| Captured stdout or stderr | 8 MiB per stream |
+| Individual scratch file size | 16 MiB |
+| Monitored scratch storage | 64 MiB |
+| Monitored scratch entries | 2,048 |
+| Scratch directory depth | 128 |
+| Open file descriptors | 64 |
+
+Evidence is streamed from gateway pages into the confined child without building another complete selection in worker memory. The child decodes the selected data under its memory limit before running the code. The execution wall clock starts after input delivery; gateway fetches keep their HTTP timeouts and remain cancellable. CPU, address-space and file-size limits apply during input decoding as well as computation. Scratch usage is monitored every 50 milliseconds, so a call can temporarily overshoot its scratch allowance. The worker's shared temporary mount supplies the hard aggregate storage ceiling, 1 GiB by default. Accounting includes unlinked open files and files retained only by memory mappings. A mapped scratch inode without an open descriptor or directory entry is conservatively charged at the individual file-size limit, which may overcount small files. Cancellation and limit failures kill and reap the child before removing its scratch directory
+
+Results include `stdout`, `stderr`, `exit_code`, `error` and `output_complete`. Nonzero interpreter exits, confinement failures and resource failures set `error` and `output_complete=false`. Available traceback output is retained. An output-size failure delivers no partial stdout/stderr; the agent can narrow its computation and retry. A successful result retains all captured output without truncation
+
+This is a process boundary sharing the worker's Linux kernel. The checked-in smoke test verifies useful Python operations, filesystem and process restrictions, raw syscall attempts, resource failures, mapping accounting, cleanup and cancellation in the actual image. Run it on the deployment's native architecture and kernel:
+
+```bash
+docker build --build-arg LITELLM_RELEASE_TAG=lens-python-test \
+  -f deploy/lens/Dockerfile -t lens-worker:python-test .
+docker run --rm --pull never --read-only --cap-drop ALL \
+  --security-opt no-new-privileges --network none \
+  --tmpfs /tmp:rw,noexec,nosuid,size=1g --entrypoint python -i \
+  lens-worker:python-test - < tests/proxy_behavior/lens/worker_python_smoke.py
+```
+
+The same checks can run through pytest by setting `LENS_TEST_WORKER_IMAGE` to an already-built native image. The worker image CI runs the standalone smoke without adding pytest to the production image

deploy/lens/compose.yaml

@@ -4,6 +4,7 @@ services:
     environment:
       LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
       LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
+      LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
     restart: unless-stopped
     read_only: true
     tmpfs:

deploy/lens/python_policy.c

@@ -0,0 +1,63 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <seccomp.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+static int allow(scmp_filter_ctx policy, const char *name)
+{
+    int number = seccomp_syscall_resolve_name(name);
+    return number < 0 ? 0 : seccomp_rule_add(policy, SCMP_ACT_ALLOW, number, 0);
+}
+
+int main(int argc, char **argv)
+{
+    const char *calls[] = {
+        "read", "write", "readv", "writev", "pread64", "pwrite64", "close", "close_range",
+        "open", "openat", "openat2", "fstat", "stat", "lstat", "newfstatat", "statx",
+        "lseek", "getdents", "getdents64", "access", "faccessat", "faccessat2",
+        "readlink", "readlinkat", "getcwd", "chdir", "fchdir", "statfs", "fstatfs",
+        "mkdir", "mkdirat", "rmdir", "unlink", "unlinkat", "rename", "renameat", "renameat2",
+        "link", "linkat", "symlink", "symlinkat", "truncate", "ftruncate", "fsync", "fdatasync",
+        "mmap", "mmap2", "mprotect", "munmap", "mremap", "madvise", "brk",
+        "rt_sigaction", "rt_sigprocmask", "rt_sigreturn", "rt_sigsuspend", "rt_sigtimedwait", "sigaltstack",
+        "getpid", "getppid", "gettid", "getuid", "geteuid", "getgid", "getegid", "getgroups",
+        "clock_gettime", "clock_getres", "clock_nanosleep", "gettimeofday", "time", "nanosleep",
+        "futex", "futex_time64", "set_tid_address", "set_robust_list", "rseq", "arch_prctl",
+        "sched_getaffinity", "sched_yield", "getrandom", "getrlimit", "setrlimit", "getrusage", "umask",
+        "dup", "dup2", "dup3", "pipe", "pipe2", "poll", "ppoll", "select", "pselect6",
+        "epoll_create", "epoll_create1", "epoll_ctl", "epoll_wait", "epoll_pwait", "epoll_pwait2",
+        "capget", "capset", "prctl", "landlock_create_ruleset", "landlock_add_rule", "landlock_restrict_self",
+        "execve", "exit", "exit_group", "uname", "sysinfo", "restart_syscall"
+    };
+    const int commands[] = {F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_SETFD, F_GETFL, F_GETLK, F_SETLK, F_SETLKW};
+    if (argc != 2) {
+        fputs("Usage: python-policy OUTPUT\n", stderr);
+        return 1;
+    }
+    scmp_filter_ctx policy = seccomp_init(SCMP_ACT_ERRNO(EPERM));
+    if (!policy)
+        return 1;
+    int result = 0;
+    for (size_t i = 0; i < sizeof(calls) / sizeof(calls[0]); i++)
+        result |= allow(policy, calls[i]);
+    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(prlimit64), 1, SCMP_A0(SCMP_CMP_EQ, 0));
+    for (size_t i = 0; i < sizeof(commands) / sizeof(commands[0]); i++)
+        result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 1, SCMP_A1(SCMP_CMP_EQ, commands[i]));
+    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 2,
+                              SCMP_A1(SCMP_CMP_EQ, F_SETFL), SCMP_A2(SCMP_CMP_MASKED_EQ, O_ASYNC, 0));
+    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIOCLEX));
+    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIONCLEX));
+    int output = open(argv[1], O_WRONLY | O_CREAT | O_TRUNC, 0444);
+    if (output < 0)
+        result = -1;
+    if (!result)
+        result = seccomp_export_bpf(policy, output);
+    if (output >= 0)
+        close(output);
+    seccomp_release(policy);
+    if (result)
+        fputs("Could not build the Python syscall policy\n", stderr);
+    return result ? 1 : 0;
+}

deploy/lens/python_runtime.py

@@ -0,0 +1,41 @@
+import json
+import subprocess
+import sys
+import sysconfig
+from itertools import chain
+from pathlib import Path
+from typing import Final
+
+
+def dependencies(path: Path, loader: Path) -> tuple[Path, ...]:
+    result: Final = subprocess.run((str(loader), "--list", str(path)), capture_output=True, text=True, check=True)
+    if "not found" in result.stdout:
+        raise RuntimeError(f"Missing Python runtime library: {path}")
+    words: Final = tuple(result.stdout.split())
+    return tuple(Path(word).resolve() for word in words if word.startswith("/"))
+
+
+def main() -> None:
+    stdlib: Final = Path(sysconfig.get_path("stdlib")).resolve()
+    executable: Final = Path(sys.executable).resolve()
+    loaders: Final = tuple(Path("/usr/lib").glob("ld-linux-*.so.*"))
+    if len(loaders) != 1:
+        raise RuntimeError("Expected one native glibc dynamic loader in the Lens worker image")
+    entries: Final = tuple(
+        path for path in stdlib.iterdir() if path.name not in ("site-packages", "dist-packages", "__pycache__")
+    )
+    extensions: Final = tuple((stdlib / "lib-dynload").glob("*.so"))
+    libraries: Final = frozenset(
+        chain.from_iterable(dependencies(binary, loaders[0]) for binary in (executable, *extensions))
+    )
+    manifest: Final = {
+        "executable": str(executable),
+        "directories": (str(stdlib),),
+        "read": tuple(sorted(str(path) for path in {*entries, *libraries})),
+        "execute": tuple(sorted(str(path) for path in (executable, *loaders))),
+    }
+    Path(sys.argv[1]).write_text(json.dumps(manifest), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

deploy/lens/stack.yaml

@@ -40,6 +40,7 @@ services:
     environment:
       LITELLM_URL: http://litellm:4000
       LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
+      LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
     depends_on: [litellm]
     networks: [proxy]
     restart: unless-stopped

docker/Dockerfile.database

@@ -77,6 +77,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
 RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
+    --group admin-mcp \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
@@ -99,6 +100,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.
 RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
+    --group admin-mcp \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \

docker/Dockerfile.non_root

@@ -81,6 +81,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
+    --group admin-mcp \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
@@ -108,6 +109,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     uv sync --frozen --no-default-groups --no-editable \
       --extra proxy \
       --extra proxy-runtime \
+      --group admin-mcp \
       --extra extra_proxy \
       --extra semantic-router \
       --extra saml \

docker/README.md

@@ -70,6 +70,32 @@ To stop the running containers, use the following command:
 docker compose down
 ```
 
+## Embedded LiteAdmin MCP
+
+Source builds containing embedded LiteAdmin MCP can serve it at `/admin/mcp` on the existing LiteLLM port. This capability is unreleased. Keep your existing database, master key, and proxy configuration, then add these settings to the serving container's environment:
+
+```bash
+LITELLM_ENABLE_ADMIN_MCP=true
+LITELLM_LICENSE="your-enterprise-license"
+PROXY_BASE_URL=https://gateway.example.com
+```
+
+For the unified source deployment described above, put them in its `.env` file and rebuild:
+
+```bash
+docker compose up -d --build
+```
+
+In componentized deployments, set the flag and license on the backend container and route `/admin/mcp` to the backend service. The gateway component excludes this endpoint. The unified, database, non-root, and backend image builds bundle the connector
+
+Hosting is disabled by default. Opting in requires a valid base Enterprise license; an unlicensed opt-in or invalid flag value prevents startup. Enabling it reserves `/admin`, so rename any MCP server alias called `admin` first
+
+With native key authentication, connect with a personal proxy-admin bearer key. When `enable_oauth2_proxy_auth` is enabled, the existing trusted-proxy identity headers select the user instead; the MCP bearer is required by the connector but does not select the native user. The resolved user must have the stored `proxy_admin` role, and `trusted_proxy_ranges` applies to the original caller's direct peer
+
+Embedded responses default to `full`; selecting `LITELLM_ADMIN_RESPONSE_VIEW=compact` requires subsequent saved-result reads to reach the same worker process, including within a multi-worker pod
+
+See the [LiteAdmin MCP guide](https://docs.litellm.ai/docs/proxy/liteadmin_mcp#run-liteadmin-mcp-inside-litellm) for client configuration, tool restrictions, and verification
+
 ## Hardened / Offline Testing
 
 To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:

docker/component_entrypoint.sh

@@ -3,7 +3,7 @@
 # stale samples from a previous container incarnation would be summed into the aggregate
 if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
     mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
-    rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db
+    rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_*
 fi
 
 case "$USE_DDTRACE" in

docker/docker-compose.tracing.yml

@@ -9,8 +9,7 @@ services:
         LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:-}
     command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
     environment:
-      LITELLM_MASTER_KEY: sk-1234
-      LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set LITELLM_MASTER_KEY}
       LITELLM_SALT_KEY: sk-local-tracing-salt-key
       DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
       STORE_MODEL_IN_DB: "True"

enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py

@@ -7,7 +7,6 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.llm_request_utils import (
     get_proxy_server_request_headers,
 )
-from litellm.proxy._types import CommonProxyErrors
 from litellm.types.utils import StandardCallbackDynamicParams
 
 
@@ -86,6 +85,7 @@ class EnterpriseCallbackControls:
     @staticmethod
     def _should_allow_dynamic_callback_disabling():
         import litellm
+        from litellm.proxy._types import CommonProxyErrors
         from litellm.proxy.proxy_server import premium_user
 
         # Check if admin has disabled this feature

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -46,6 +46,8 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
     WebhookEvent,
 )
+from litellm.repositories.table_repositories import InvitationLinkRepository
+from litellm.repositories.user_repository import UserRepository
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
 
@@ -844,7 +846,7 @@ class BaseEmailLogger(CustomLogger):
             )
             return None
 
-        user_row = await prisma_client.db.litellm_usertable.find_unique(
+        user_row = await UserRepository(prisma_client).table.find_unique(
             where={"user_id": user_id}
         )
 
@@ -929,7 +931,7 @@ class BaseEmailLogger(CustomLogger):
         try:
             # Try to get existing invitation
             existing_invitations = (
-                await prisma_client.db.litellm_invitationlink.find_many(
+                await InvitationLinkRepository(prisma_client).table.find_many(
                     where={"user_id": user_id},
                     order={"created_at": "desc"},
                 )

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -15,6 +15,10 @@ from litellm.constants import (
     MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
     MAX_OBJECTS_PER_POLL_CYCLE,
 )
+from litellm.repositories.table_repositories import ManagedObjectRepository
+from litellm.repositories.team_repository import TeamRepository
+from litellm.repositories.user_repository import UserRepository
+from litellm.repositories.verification_token_repository import VerificationTokenRepository
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
@@ -58,25 +62,19 @@ class _ManagedObjectRow(Protocol):
 
 
 def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
-    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
-    return table
+    return ManagedObjectRepository(prisma_client).table
 
 
 def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
-    table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
-    return table
+    return UserRepository(prisma_client).table
 
 
 def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
-    table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
-        prisma_client.db.litellm_verificationtoken
-    )
-    return table
+    return VerificationTokenRepository(prisma_client).table
 
 
 def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
-    table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
-    return table
+    return TeamRepository(prisma_client).table
 
 
 class CheckBatchCost:

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -16,6 +16,7 @@ from litellm.constants import (
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
+from litellm.repositories.table_repositories import ManagedObjectRepository
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
@@ -43,8 +44,7 @@ class _ManagedObjectRow(Protocol):
 
 
 def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
-    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
-    return table
+    return ManagedObjectRepository(prisma_client).table
 
 
 class CheckResponsesCost:

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -1647,7 +1647,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
         owner_filter: Final = build_owner_filter(user_api_key_dict)
         if owner_filter is None:
-            return FileListPage(**build_list_page([]))
+            return FileListPage.model_validate(build_list_page([]))
 
         if after:
             cursor_row = await _managed_file_table(self.prisma_client).find_first(
@@ -1686,7 +1686,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             cursor_id = chunk[-1].unified_file_id
             chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
 
-        return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
+        return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
 
     def _is_batch_polling_enabled(self) -> bool:
         """

enterprise/litellm_enterprise/proxy/liteadmin.py

@@ -15,11 +15,12 @@ from urllib.parse import urlencode, urlsplit
 import httpx
 from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import HTMLResponse, RedirectResponse, Response
-from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
+from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
 
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._experimental.mcp_server.oauth_utils import get_request_base_url
 from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
+from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.proxy.auth.auth_checks import UserNotFoundError
 
 router: Final = APIRouter()
@@ -34,14 +35,14 @@ _HEADERS: Final = {
 }
 
 
-class LinkDetails(BaseModel):
+class LinkDetails(LiteLLMBaseModel):
     model_config = ConfigDict(frozen=True, strict=True, extra="forbid")
     workspace_id: str = Field(min_length=1, max_length=64)
     slack_user_id: str = Field(min_length=1, max_length=64)
     email: str = Field(min_length=1, max_length=320)
 
 
-class AdminSession(BaseModel):
+class AdminSession(LiteLLMBaseModel):
     model_config = ConfigDict(frozen=True)
     user_id: str
     credential: SecretStr

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,7 +22,7 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management.teams.access import is_team_admin
+from litellm.proxy.management.teams.authz import is_team_admin
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
@@ -433,7 +433,7 @@ async def new_project(
 
     ```bash
     curl --location 'http://0.0.0.0:4000/project/new' \\
-    --header 'Authorization: Bearer sk-1234' \\
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
     --header 'Content-Type: application/json' \\
     --data '{
         "project_alias": "flight-search-assistant",
@@ -460,7 +460,7 @@ async def new_project(
 
     ```bash
     curl --location 'http://0.0.0.0:4000/project/new' \\
-    --header 'Authorization: Bearer sk-1234' \\
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
     --header 'Content-Type: application/json' \\
     --data '{
         "project_alias": "hotel-recommendations",
@@ -648,7 +648,7 @@ async def update_project(
     Example:
     ```bash
     curl --location 'http://0.0.0.0:4000/project/update' \\
-    --header 'Authorization: Bearer sk-1234' \\
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
     --header 'Content-Type: application/json' \\
     --data '{
         "project_id": "project-123",
@@ -876,7 +876,7 @@ async def delete_project(
     Example:
     ```bash
     curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \\
-    --header 'Authorization: Bearer sk-1234' \\
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
     --header 'Content-Type: application/json' \\
     --data '{
         "project_ids": ["project-123", "project-456"]
@@ -983,7 +983,7 @@ async def project_info(
     Example:
     ```bash
     curl --location 'http://0.0.0.0:4000/project/info?project_id=project-123' \\
-    --header 'Authorization: Bearer sk-1234'
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY"
     ```
     """
     from litellm.proxy.proxy_server import prisma_client
@@ -1052,7 +1052,7 @@ async def list_projects(
     Example:
     ```bash
     curl --location 'http://0.0.0.0:4000/project/list' \\
-    --header 'Authorization: Bearer sk-1234'
+    --header "Authorization: Bearer $LITELLM_MASTER_KEY"
     ```
     """
     from litellm.proxy.proxy_server import prisma_client

enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py

@@ -1,12 +1,13 @@
 import enum
 from typing import Dict, List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from litellm.proxy._types import WebhookEvent
+from litellm.types.llms.base import LiteLLMBaseModel
 
 
-class EmailParams(BaseModel):
+class EmailParams(LiteLLMBaseModel):
     logo_url: str
     support_contact: str
     base_url: str
@@ -39,14 +40,14 @@ class EmailEvent(str, enum.Enum):
     soft_budget_crossed = "Soft Budget Crossed"
     max_budget_alert = "Max Budget Alert"
 
-class EmailEventSettings(BaseModel):
+class EmailEventSettings(LiteLLMBaseModel):
     event: EmailEvent
     enabled: bool
-class EmailEventSettingsUpdateRequest(BaseModel):
+class EmailEventSettingsUpdateRequest(LiteLLMBaseModel):
     settings: List[EmailEventSettings]
-class EmailEventSettingsResponse(BaseModel):
+class EmailEventSettingsResponse(LiteLLMBaseModel):
     settings: List[EmailEventSettings]
-class DefaultEmailSettings(BaseModel):
+class DefaultEmailSettings(LiteLLMBaseModel):
     """Default settings for email events"""
     settings: Dict[EmailEvent, bool] = Field(
         default_factory=lambda: {

enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py

@@ -1,10 +1,12 @@
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
+
+from litellm.types.llms.base import LiteLLMBaseModel
 
 
-class AuditLogResponse(BaseModel):
+class AuditLogResponse(LiteLLMBaseModel):
     """Response model for a single audit log entry"""
 
     id: str
@@ -18,7 +20,7 @@ class AuditLogResponse(BaseModel):
     updated_values: Optional[Dict[str, Any]] = None
 
 
-class PaginatedAuditLogResponse(BaseModel):
+class PaginatedAuditLogResponse(LiteLLMBaseModel):
     """Response model for paginated audit logs"""
 
     audit_logs: List[AuditLogResponse]

gateway/Dockerfile

@@ -60,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
+        --group admin-mcp \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
@@ -72,6 +73,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
+        --group admin-mcp \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \

gateway/routes/allowlist.py

@@ -76,6 +76,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/containers",
     "/v1/evals",
     "/v1/traces",
+    "/v1/logs",
     "/v1/memory",
     "/queue/chat/",
     # Google data plane (v1beta is the Google AI Studio version)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_add_end_user_models/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_EndUserTable" ADD COLUMN IF NOT EXISTS "models" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230000_lens_trace_findings_indexes/migration.sql

@@ -0,0 +1,3 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_LensRun_completed_executions_idx"
+ON "LiteLLM_LensRun" USING GIN ((data->'sample'->'executions') jsonb_path_ops)
+WHERE data->>'status'='completed';

litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230100_lens_current_findings_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_jobs_idx"
+ON "LiteLLM_Lens" USING GIN ((data->'jobs') jsonb_path_ops);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -656,6 +656,7 @@ model LiteLLM_EndUserTable {
   spend      Float    @default(0.0)
   allowed_model_region String? // require all user requests to use models in this specific region
   default_model String? // use along with 'allowed_model_region'. if no available model in region, default to this model.
+  models String[] @default([])
   budget_id String?
   object_permission_id String?
   litellm_budget_table LiteLLM_BudgetTable?   @relation(fields: [budget_id], references: [budget_id])

litellm-rust/.agents/skills/rust-string-enums/SKILL.md

@@ -0,0 +1,66 @@
+---
+name: rust-string-enums
+description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
+---
+
+# Rust string enums
+
+Use this skill when adding or changing enums represented by a single string, or surveying handwritten string conversions
+
+## Choose the representation
+
+For an enum with fixed spellings and an unknown-string fallback, prefer `strum::EnumString` and `strum::Display` together with `serde_with::DeserializeFromStr` and `serde_with::SerializeDisplay`. Keep each wire spelling in the Strum attributes instead of repeating it in a handwritten Serde match
+
+Strum implements string conversion traits, not Serde traits. `EnumString` implements `FromStr`; `Display` formats the wire string. The two `serde_with` derives connect those traits to Serde. `AsRefStr` provides a borrowed string accessor and is optional
+
+```rust
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    strum::AsRefStr,
+    strum::EnumString,
+    strum::Display,
+    serde_with::DeserializeFromStr,
+    serde_with::SerializeDisplay,
+)]
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
+#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
+pub enum EventType {
+    #[strum(serialize = "event.created")]
+    Created,
+    #[strum(default, transparent)]
+    Other(String),
+}
+```
+
+Use `#[strum(serialize_all = "snake_case")]` or another supported case style when it exactly matches the contract. Use explicit variant spellings otherwise. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
+
+Use only the derives the contract needs. A deserialize-only type should remain deserialize-only. Do not add an unknown variant to a closed enum, or derive Serde for a type that currently has no serialization contract
+
+Plain Serde derives with `rename` or `rename_all` remain appropriate for closed unit enums. Adding Strum and serde_with solely to replace working Serde derives adds little value. When a type has both Serde and Strum parsing, compare their accepted inputs before sharing the parser: case-insensitive Strum parsing must not silently make strict JSON parsing case-insensitive
+
+## Preserve behavior during migration
+
+Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion APIs. Keep conversions needed by callers or compatibility even when Serde no longer uses them
+
+Use the workspace dependencies and enable `serde_with.workspace = true` in a crate only when needed. Check the versions and enabled features in `Cargo.toml` and `Cargo.lock` rather than upgrading dependencies for this refactor
+
+Remove replaced manual Serde implementations and obsolete Serde conversion attributes. Do not combine the new derives with `wire_type`, `request_type`, or `response_type` aliases that already derive the same Serde traits. Expand the necessary non-Serde derives and schema attributes locally rather than changing shared aliases for unrelated types
+
+Preserve the generated schema, including titles and definition names. Open string enums need a string schema, including unknown values. When replacing Serde `from`/`into` attributes that previously supplied that schema, retain their schema behavior with `#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]` and compare the full generated result. `schemars(with = "String")` also makes a string schema, but changes the schema name and title, so keep it only where it already matches the contract
+
+Keep custom `FromStr` and `Display` implementations for structured strings or validation that Strum does not express faithfully. Their Serde adapters can still use `DeserializeFromStr` and `SerializeDisplay`. Do not replace JSON visitors, tagged payload enums, permissive value wrappers, or domain transformations with string parsing
+
+For a requested survey, document candidates and exceptions without migrating source. If the user asks to approve a bulk migration, present the concrete scope and wait for that approval
+
+## Verify the contract
+
+Extend existing mapped tests with named `rstest` cases. Assert both parsing into the expected variant and serialization to the expected wire string. Include unknown and empty strings for open enums, accepted aliases when present, and rejection of non-string JSON. A decode-only assertion does not prove a round trip
+
+Test structured parsers with valid and invalid payloads, including their existing error behavior. For types with schema support, check the string schema and run the affected crate tests with the schema feature enabled. Run affected downstream checks when conversion APIs or derive aliases change
+
+## Upstream references
+
+The workspace used Strum 0.28.0 and serde_with 3.16.1 when this guidance was written. Consult the matching version of the [EnumString docs](https://docs.rs/strum/0.28.0/strum/derive.EnumString.html), [Display docs](https://docs.rs/strum_macros/0.28.0/strum_macros/derive.Display.html), [DeserializeFromStr docs](https://docs.rs/serde_with/3.16.1/serde_with/derive.DeserializeFromStr.html), and [SerializeDisplay docs](https://docs.rs/serde_with/3.16.1/serde_with/derive.SerializeDisplay.html). Schemars documents [schema overrides and Serde conversion attributes](https://docs.rs/schemars/1.2.2/schemars/derive.JsonSchema.html)

litellm-rust/AGENTS.md

@@ -2,6 +2,8 @@
 
 For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.agents/skills/rust-tracing/SKILL.md)
 
+For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md)
+
 ## Test placement
 
 - Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`

litellm-rust/Cargo.lock

@@ -3697,50 +3697,6 @@ dependencies = [
  "thiserror 2.0.19",
 ]
 
-[[package]]
-name = "litellm-core"
-version = "0.1.0"
-dependencies = [
- "base64 0.22.1",
- "bytes",
- "futures-util",
- "litellm-auth",
- "litellm-auth-aws",
- "litellm-auth-gcp",
- "litellm-cache",
- "litellm-cache-memory",
- "litellm-cache-response",
- "litellm-core-utils",
- "litellm-framing",
- "litellm-host",
- "litellm-host-native",
- "litellm-http",
- "litellm-llms",
- "litellm-llms-types",
- "litellm-secrets",
- "litellm-tracing",
- "mime_guess",
- "moka",
- "rand 0.8.7",
- "reqwest 0.12.28",
- "rstest",
- "rstest_reuse",
- "serde",
- "serde_json",
- "sha2 0.10.9",
- "strum",
- "subtle",
- "thiserror 2.0.19",
- "time",
- "tokio",
- "tokio-tungstenite",
- "tokio-util",
- "tracing",
- "url",
- "veil",
- "wiremock",
-]
-
 [[package]]
 name = "litellm-core-utils"
 version = "0.1.0"
@@ -3796,7 +3752,7 @@ dependencies = [
 ]
 
 [[package]]
-name = "litellm-framing"
+name = "litellm-framer"
 version = "0.1.0"
 dependencies = [
  "aws-smithy-eventstream",
@@ -3821,12 +3777,12 @@ dependencies = [
  "http-body-util",
  "litellm-auth-types",
  "litellm-config",
- "litellm-core",
  "litellm-gateway-auth",
  "litellm-gateway-inference",
  "litellm-gateway-mcp",
  "litellm-gateway-ui",
  "litellm-http",
+ "litellm-inference",
  "litellm-llms",
  "litellm-secrets",
  "litellm-tracing",
@@ -3875,11 +3831,16 @@ dependencies = [
  "litellm-auth",
  "litellm-cache-memory",
  "litellm-cache-response",
- "litellm-core",
  "litellm-gateway-auth",
  "litellm-host",
  "litellm-host-http",
  "litellm-http",
+ "litellm-inference",
+ "litellm-inference-chat",
+ "litellm-inference-messages",
+ "litellm-inference-ocr",
+ "litellm-inference-responses",
+ "litellm-inference-transcription",
  "litellm-llms",
  "litellm-llms-types",
  "litellm-router",
@@ -4038,6 +3999,178 @@ dependencies = [
  "webpki-roots",
 ]
 
+[[package]]
+name = "litellm-inference"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-cache",
+ "litellm-cache-memory",
+ "litellm-cache-response",
+ "litellm-core-utils",
+ "litellm-framer",
+ "litellm-host",
+ "litellm-http",
+ "litellm-llms",
+ "litellm-secrets",
+ "litellm-tracing",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.19",
+ "time",
+ "tokio",
+ "tokio-util",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "litellm-inference-chat"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-auth",
+ "litellm-cache-memory",
+ "litellm-cache-response",
+ "litellm-core-utils",
+ "litellm-host",
+ "litellm-host-native",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-inference-testing",
+ "litellm-llms",
+ "litellm-llms-types",
+ "litellm-secrets",
+ "litellm-tracing",
+ "rstest",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "wiremock",
+]
+
+[[package]]
+name = "litellm-inference-messages"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-auth-aws",
+ "litellm-cache-memory",
+ "litellm-cache-response",
+ "litellm-core-utils",
+ "litellm-host",
+ "litellm-host-native",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-inference-testing",
+ "litellm-llms",
+ "litellm-llms-types",
+ "litellm-secrets",
+ "litellm-tracing",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "wiremock",
+]
+
+[[package]]
+name = "litellm-inference-ocr"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-auth-aws",
+ "litellm-auth-gcp",
+ "litellm-core-utils",
+ "litellm-host",
+ "litellm-host-native",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-inference-testing",
+ "litellm-llms",
+ "litellm-llms-types",
+ "litellm-secrets",
+ "litellm-tracing",
+ "mime_guess",
+ "rand 0.8.7",
+ "rstest",
+ "serde",
+ "serde_json",
+ "time",
+ "tokio",
+ "tracing",
+ "wiremock",
+]
+
+[[package]]
+name = "litellm-inference-responses"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-cache",
+ "litellm-cache-memory",
+ "litellm-cache-response",
+ "litellm-host",
+ "litellm-host-native",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-inference-testing",
+ "litellm-llms",
+ "litellm-llms-types",
+ "litellm-secrets",
+ "litellm-tracing",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde_json",
+ "tokio",
+ "tokio-tungstenite",
+ "tracing",
+ "wiremock",
+]
+
+[[package]]
+name = "litellm-inference-testing"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-secrets",
+]
+
+[[package]]
+name = "litellm-inference-transcription"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-auth",
+ "litellm-core-utils",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-inference-testing",
+ "litellm-llms",
+ "litellm-secrets",
+ "litellm-tracing",
+ "rstest",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-llms"
 version = "0.1.0"
@@ -4053,7 +4186,7 @@ dependencies = [
  "litellm-auth-azure",
  "litellm-auth-gcp",
  "litellm-core-utils",
- "litellm-framing",
+ "litellm-framer",
  "litellm-host",
  "litellm-http",
  "litellm-llms-types",
@@ -4085,26 +4218,6 @@ dependencies = [
  "strum",
 ]
 
-[[package]]
-name = "litellm-migrate"
-version = "0.1.0"
-dependencies = [
- "litellm-migrate-macros",
- "rstest",
-]
-
-[[package]]
-name = "litellm-migrate-macros"
-version = "0.1.0"
-dependencies = [
- "proc-macro2",
- "quote",
- "rstest",
- "syn 2.0.119",
- "tempfile",
- "thiserror 2.0.19",
-]
-
 [[package]]
 name = "litellm-model-catalog"
 version = "0.1.0"
@@ -4132,22 +4245,20 @@ dependencies = [
  "litellm-auth",
  "litellm-auth-aws",
  "litellm-cache",
- "litellm-cache-azure-blob",
- "litellm-cache-disk",
- "litellm-cache-gcs",
  "litellm-cache-memory",
- "litellm-cache-qdrant-semantic",
  "litellm-cache-redis",
- "litellm-cache-redis-semantic",
  "litellm-cache-response",
- "litellm-cache-s3",
- "litellm-cache-valkey-semantic",
  "litellm-callbacks-legacy-python",
- "litellm-core",
  "litellm-core-utils",
  "litellm-host",
  "litellm-host-python",
  "litellm-http",
+ "litellm-inference",
+ "litellm-inference-chat",
+ "litellm-inference-messages",
+ "litellm-inference-ocr",
+ "litellm-inference-responses",
+ "litellm-inference-transcription",
  "litellm-llms",
  "litellm-llms-types",
  "litellm-secrets",
@@ -4156,25 +4267,24 @@ dependencies = [
  "litellm-storage-clickhouse",
  "litellm-token-counter",
  "litellm-traces",
+ "litellm-traces-cache",
  "litellm-traces-clickhouse",
  "litellm-tracing",
  "prost",
  "pyo3",
  "pyo3-async-runtimes",
- "qdrant-client",
  "redis",
  "reqwest 0.12.28",
  "rstest",
  "serde",
  "serde_json",
  "serde_with",
- "sha2 0.10.9",
+ "sqlx",
  "strum",
  "thiserror 2.0.19",
  "tokio",
  "tokio-tungstenite",
  "tracing",
- "url",
  "veil",
  "wiremock",
 ]
@@ -4199,7 +4309,8 @@ name = "litellm-router"
 version = "0.1.0"
 dependencies = [
  "litellm-config",
- "litellm-core",
+ "litellm-inference",
+ "litellm-inference-messages",
  "rstest",
 ]
 
@@ -4367,6 +4478,9 @@ dependencies = [
  "rstest",
  "serde",
  "serde_json",
+ "serde_with",
+ "sqlx",
+ "testcontainers-modules",
  "thiserror 2.0.19",
  "tokio",
  "url",
@@ -4465,6 +4579,8 @@ dependencies = [
  "schemars 1.2.2",
  "serde",
  "serde_json",
+ "serde_with",
+ "sha2 0.10.9",
  "strum",
  "thiserror 2.0.19",
  "time",
@@ -4474,13 +4590,17 @@ dependencies = [
 name = "litellm-traces-cache"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "litellm-traces",
  "moka",
  "rstest",
+ "serde",
  "serde_json",
  "sha2 0.10.9",
  "thiserror 2.0.19",
+ "time",
  "tokio",
+ "tracing",
 ]
 
 [[package]]
@@ -4488,14 +4608,11 @@ name = "litellm-traces-clickhouse"
 version = "0.1.0"
 dependencies = [
  "askama",
- "base64 0.22.1",
  "flate2",
  "futures-util",
  "hmac 0.12.1",
- "itertools 0.14.0",
  "jsonschema",
  "litellm-http",
- "litellm-migrate",
  "litellm-storage-clickhouse",
  "litellm-traces",
  "litellm-traces-cache",
@@ -4506,12 +4623,12 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.10.9",
+ "sqlx",
  "strum",
  "testcontainers-modules",
  "thiserror 2.0.19",
  "time",
  "tokio",
- "tracing",
  "url",
  "wiremock",
 ]
@@ -6068,17 +6185,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "rstest_reuse"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14"
-dependencies = [
- "quote",
- "rand 0.8.7",
- "syn 2.0.119",
-]
-
 [[package]]
 name = "rusqlite"
 version = "0.39.0"

litellm-rust/Cargo.toml

@@ -16,9 +16,13 @@ litellm-traces = { path = "crates/traces" }
 litellm-traces-cache = { path = "crates/traces-cache" }
 litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
 litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
-litellm-migrate = { path = "crates/migrate" }
-litellm-migrate-macros = { path = "crates/migrate-macros" }
-litellm-core = { path = "crates/core" }
+litellm-inference = { path = "crates/inference" }
+litellm-inference-transcription = { path = "crates/inference-transcription" }
+litellm-inference-responses = { path = "crates/inference-responses" }
+litellm-inference-messages = { path = "crates/inference-messages" }
+litellm-inference-chat = { path = "crates/inference-chat" }
+litellm-inference-ocr = { path = "crates/inference-ocr" }
+litellm-inference-testing = { path = "crates/inference-testing" }
 litellm-gateway-mcp = { path = "crates/gateway-mcp" }
 litellm-gateway = { path = "crates/gateway" }
 litellm-gateway-inference = { path = "crates/gateway-inference" }
@@ -30,7 +34,7 @@ litellm-host = { path = "crates/host" }
 litellm-host-http = { path = "crates/host-http" }
 litellm-host-native = { path = "crates/host-native" }
 litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
-litellm-framing = { path = "crates/framer" }
+litellm-framer = { path = "crates/framer" }
 litellm-auth = { path = "crates/auth" }
 litellm-auth-types = { path = "crates/auth-types" }
 litellm-auth-aws = { path = "crates/auth-aws" }

litellm-rust/crates/core/AGENTS.md

@@ -1,49 +0,0 @@
-litellm-core owns route orchestration. Messages and HTTP Responses return `litellm_host::call::CallOutput`, containing either a completed response or a stream head and chunks. OCR and currently non-streaming Chat Completions return their completed response directly
-
-Hosts assemble route objects from shared `CoreResources`, HTTP settings, and secret sources. Each route owns its provider client and authentication dependencies. Gateway routes live for the gateway lifetime; Python assembles routes per call from its settings snapshot
-
-Chat Completions, Messages, Responses, and OCR execute through their route objects. Calls pass `Interceptors` and an optional `ObservationSender` separately; use `&()` for no hooks and `None` for no observer. Construction does no work; preparation and lifecycle observation begin when the future is polled. Handlers accept `Interceptors`, never a concrete `ChannelInterceptors`. Native observers receive start and terminal events through the shared call runner; a stream retains its lifecycle until exhaustion, error, or drop. Hosted routes leave terminal observation to their driver
-
-`route.rs` declares the concrete `Protocol` and implements a route method that accepts a typed request and constructs a `litellm_host::call::HostedMachine` with `hosted_call`. The shared call plumbing owns stream opening, delivery, backpressure, and detachment. Request decoding belongs to the boundary before the machine starts. Route closures only supply execution dependencies and route-specific host capabilities such as an OCR token provider. Use `run_hosted` for a native host so detachment is reported as cancellation. Python uses its own shared driver and preserves caller-task callback execution
-
-Responses WebSocket sessions remain separate from the HTTP call driver because a connection can accept multiple requests while receiving events
-
-## Crate layering
-
-For Messages, Responses, Chat Completions, OCR, and other API formats, `core/src/<format>/` owns orchestration. Shared API data contracts belong in `litellm-llms-types`, adapter contracts and shared transformation machinery in `llms/src/base_llm/<format>/`, and provider policy in `llms/src/<provider>/<format>/`. A repeated format directory name does not imply interchangeable responsibilities. Select concrete adapters here, then invoke their contracts instead of applying one provider's policy to every call. Route types describe call envelopes and execution state, not duplicate public payload schemas
-
-Crates separate API data, transformations, transport, and orchestration. Python package names identify counterparts, not ownership. Dependencies only point down:
-
-- `litellm-llms-types` owns shared inference API contracts, grouped by format: pure serde data and shape validation, no I/O
-- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments, settings lookup and layer merge), no network I/O
-- `litellm-http` is Rust-only and route-neutral: settings resolution, the pooled `reqwest` clients, TLS, proxies, the SSRF-safe media fetcher, request and header helpers, and transport errors. Python's `litellm/llms/custom_httpx/` is split by responsibility instead of mirrored: its transport half lives here, its OCR handler in `litellm-llms`
-- `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `base_llm/ocr/handler.rs` (the OCR request handler)
-- `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks
-
-A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, the provider-level hooks OCR implements over its host until it folds into `litellm_host::interceptors::Interceptors`. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
-
-## Error placement
-
-The workspace `Error definitions` rules shape each crate's error; this section decides which crate and module a failure belongs to
-
-A failure is declared once, by the lowest crate that raises it. Every crate above nests that error unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`. `RouteError` collects route failures and never re-declares a variant a lower crate raises
-
-Scope follows the concept, not the first caller. An error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it. A failure two providers or two routes can hit, such as wire framing, stream event decoding, or a malformed provider response, belongs to the crate that owns the concept: `litellm-framing` for framing, `litellm_llms::Error` for the transformation layer
-
-`litellm_llms::Error` (`crates/llms/src/error.rs`) is the one transformation error for every provider and API. `base_llm/ocr/error.rs` is the recorded exception until OCR folds into it
-
-Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business.
-
-## Response caching and accounting boundary
-
-Attach a `litellm_cache_response::ScopedCache` with `route.with_cache(cache)`. Cached and uncached routes use the same `execute` and `machine` methods. `CallOptions` carries a scope-free `CachePolicy` and observation; per-call policy never replaces the attached scope or service
-
-Messages groups per-call dependencies in `CallContext` and explicitly sequences cache lookup, provider execution, result acceptance, and cache storage. Provider transport does not own cache orchestration. Stream capture remains in the shared cache implementation
-
-Core owns request identity, typed response reconstruction and stream capture/replay. `cache-response` owns cache policy, namespacing, scope encoding, versioned envelopes and freshness. The SDK explicitly chooses shared scope. The gateway derives isolated scope from authenticated identity before attaching its service
-
-Core delivers `ExecutionFacts` through the awaited `ResultReady` host operation for both provider and cached results, before public response processing or stream opening. Facts carry resolved model/provider and result source, including the hit key. Usage remains in the typed response or delivered stream, where completion and cancellation determine what was actually reported. Passive observation is not an accounting delivery mechanism
-
-Core does not calculate prices, charge budgets, or update rate-limit counters. The legacy Python callback adapter translates execution facts into the existing Python logging contract; Python remains the accounting owner on that path. Native gateway accounting belongs to gateway dependencies, independently of `host-python`. Response-cache services expose no coordination counters or reservation APIs. A shared Redis deployment does not make response storage and accounting coordination the same dependency
-
-Cache lookup follows provider preparation, credential resolution and the request interceptor. Keys describe the effective provider URL, authenticated headers and rewritten body. Signed requests bypass caching until the signing identity has a stable cache representation

litellm-rust/crates/core/src/constants.rs

@@ -1,15 +0,0 @@
-pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com";
-
-/// Full-request timeout ceiling for Anthropic Messages provider calls, in
-/// seconds. Mirrors the Python Anthropic Messages default. The per-request
-/// timeout from the caller still overrides this on the request builder.
-pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
-
-/// Full-request timeout ceiling for chat completions provider calls, in
-/// seconds. Mirrors the Python chat completions default.
-pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600;
-
-pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600;
-
-/// `object` field every non-streaming chat completion response carries.
-pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion";

litellm-rust/crates/core/src/messages/AGENTS.md

@@ -1,7 +0,0 @@
-This directory owns provider-independent Messages call orchestration: the entrypoint, call envelopes, provider selection, credential resolution, transport coordination, hooks, and stream lifecycle. Shared API data contracts belong in `litellm-llms-types::formats::messages`, adapter contracts and execution inputs in `llms/src/base_llm/messages`, and provider implementations in `llms/src/<provider>/messages`
-
-Select concrete provider adapters and invoke their contracts. Delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters. Keep provider policy out of request preparation and transport handlers. Calling a concrete provider helper for every provider is still a policy dependency
-
-Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
-
-Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior. Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

litellm-rust/crates/framer/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "litellm-framing"
+name = "litellm-framer"
 version = "0.1.0"
 edition.workspace = true
 license.workspace = true

litellm-rust/crates/framer/tests/aws_event_stream.rs

@@ -6,7 +6,7 @@ use std::io;
 
 use bytes::Bytes;
 use futures_util::{StreamExt, TryStreamExt, stream};
-use litellm_framing::{
+use litellm_framer::{
     EventStreamError,
     aws_event_stream::{AwsEventStreamCodec, Header, HeaderValue, Message},
     frames,

litellm-rust/crates/framer/tests/chaining.rs

@@ -4,7 +4,7 @@ mod support;
 
 use bytes::Bytes;
 use futures_util::{StreamExt, TryStreamExt};
-use litellm_framing::{
+use litellm_framer::{
     EventStreamError, SseError,
     aws_event_stream::{AwsEventStreamCodec, Message},
     frames,

litellm-rust/crates/framer/tests/sse.rs

@@ -6,7 +6,7 @@ use std::io;
 
 use bytes::Bytes;
 use futures_util::{StreamExt, TryStreamExt, stream};
-use litellm_framing::{
+use litellm_framer::{
     SseError, frames,
     sse::{SseCodec, SseEvent},
 };

litellm-rust/crates/gateway-auth/tests/auth.rs

@@ -94,8 +94,8 @@ async fn enforces_configured_keys_without_exposing_secrets(
 #[rstest]
 fn hash_token_matches_python_sha256_hexdigest() {
     assert_eq!(
-        hash_token("sk-1234"),
-        "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"
+        hash_token("sk-9876"),
+        "595b23af2e99cee580245f388d3244a39247a25a3846e898e9c78841f8471a3e"
     );
 }
 

litellm-rust/crates/gateway-inference/AGENTS.md

@@ -1,6 +1,6 @@
 - Expose a mountable Axum router; listener binding, server lifecycle, and shared inbound middleware belong to `gateway`
 - Own endpoint paths, request parsing, model alias resolution, and API-specific response and SSE error formats; delegate hosted call execution and HTTP body delivery to host-http
-- Delegate inference execution to `core` and provider transformations and authentication to `llms` and the auth crates; do not duplicate them in handlers
-- Let core validate inference fields and supported features, then map its errors to HTTP responses; do not add gateway checks for temporary core limitations
+- Delegate inference execution to the `inference-<fmt>` crates and provider transformations and authentication to `llms` and the auth crates; do not duplicate them in handlers
+- Let the `inference-<fmt>` crates validate inference fields and supported features, then map their errors to HTTP responses; do not add gateway checks for temporary inference limitations
 - Use injected deployments, HTTP pools, settings, and secret sources; do not load process configuration or construct independent clients in handlers
-- Test HTTP contracts here, including status codes, forwarded headers, error envelopes, and streaming behavior; keep core and provider tests in their owning crates
+- Test HTTP contracts here, including status codes, forwarded headers, error envelopes, and streaming behavior; keep inference and provider tests in their owning crates

litellm-rust/crates/gateway-inference/Cargo.toml

@@ -12,7 +12,12 @@ base64.workspace = true
 bytes.workspace = true
 litellm-auth.workspace = true
 litellm-gateway-auth.workspace = true
-litellm-core.workspace = true
+litellm-inference.workspace = true
+litellm-inference-transcription.workspace = true
+litellm-inference-responses.workspace = true
+litellm-inference-messages.workspace = true
+litellm-inference-chat.workspace = true
+litellm-inference-ocr.workspace = true
 litellm-host-http.workspace = true
 litellm-host.workspace = true
 litellm-http.workspace = true

litellm-rust/crates/gateway-inference/src/audio_transcription.rs

@@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc};
 
 use axum::{Json, extract::State, response::IntoResponse};
 use base64::{Engine, engine::general_purpose::STANDARD};
-use litellm_core::audio_transcription::types::AudioTranscriptionRequest;
+use litellm_inference_transcription::types::AudioTranscriptionRequest;
 use serde_json::{Value, json};
 
 use crate::{

litellm-rust/crates/gateway-inference/src/caching.rs

@@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
 #[derive(Clone, Default)]
 pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
 
-impl litellm_host::interceptors::Interceptors<litellm_core::RouteError> for CacheHeaders {
+impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
     async fn before_provider_request(
         &self,
         wire: litellm_host::interceptors::WireRequest,
         _: litellm_host::interceptors::RequestContext,
-    ) -> Result<litellm_host::interceptors::WireRequest, litellm_core::RouteError> {
+    ) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
         Ok(wire)
     }
 
     async fn after_provider_response(
         &self,
         _: litellm_host::interceptors::RawResponse,
-    ) -> Result<(), litellm_core::RouteError> {
+    ) -> Result<(), litellm_inference::RouteError> {
         Ok(())
     }
 
     async fn result_ready(
         &self,
         facts: litellm_host::interceptors::ExecutionFacts,
-    ) -> Result<(), litellm_core::RouteError> {
+    ) -> Result<(), litellm_inference::RouteError> {
         if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
             let _ = self.0.set(key);
         }

litellm-rust/crates/gateway-inference/src/chat_completions.rs

@@ -6,7 +6,7 @@ use axum::{
     extract::{Path, State},
     response::{IntoResponse, Response},
 };
-use litellm_core::chat_completions::types::ChatCompletionsCall;
+use litellm_inference_chat::types::ChatCompletionsCall;
 use serde_json::{Map, Value};
 
 use crate::{Error, Gateway, JsonObject, request};

litellm-rust/crates/gateway-inference/src/error.rs

@@ -3,8 +3,8 @@ use axum::{
     Json,
     response::{IntoResponse, Response},
 };
-use litellm_core::RouteError;
 use litellm_http::transport::Error as TransportError;
+use litellm_inference::RouteError;
 use litellm_llms::base_llm::ocr::error::Error as OcrError;
 use serde_json::{Map, Value, json};
 

litellm-rust/crates/gateway-inference/src/lib.rs

@@ -15,11 +15,13 @@ mod responses;
 use std::sync::Arc;
 
 use axum::{Router, routing::post};
-use litellm_core::{
-    audio_transcription::AudioTranscriptionRoute, chat_completions::ChatCompletionsRoute,
-    messages::MessagesRoute, ocr::OcrRoute, resources::CoreResources, responses::ResponsesRoute,
-};
 use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy};
+use litellm_inference::resources::CoreResources;
+use litellm_inference_chat::ChatCompletionsRoute;
+use litellm_inference_messages::MessagesRoute;
+use litellm_inference_ocr::OcrRoute;
+use litellm_inference_responses::ResponsesRoute;
+use litellm_inference_transcription::AudioTranscriptionRoute;
 use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
 use litellm_secrets::source::SecretSource;
 

litellm-rust/crates/gateway-inference/src/messages.rs

@@ -10,8 +10,8 @@ use axum::{
     http::HeaderMap,
     response::{IntoResponse, Response},
 };
-use litellm_core::messages::{MessagesCall, messages_body, route::Messages};
 use litellm_host_http::Sse;
+use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
 use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
 use serde_json::{Map, Value};
 

litellm-rust/crates/gateway-inference/src/ocr.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use axum::{Json, extract::State, http::HeaderMap, response::IntoResponse};
 use litellm_auth::SecretValue;
-use litellm_core::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
+use litellm_inference_ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
 use litellm_llms::base_llm::ocr::transformation::decode_request_value;
 use litellm_llms_types::formats::ocr::OcrDocument;
 use serde_json::Value;

litellm-rust/crates/gateway-inference/src/responses.rs

@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
 use axum::{Json, body::Bytes, extract::State, response::Response};
-use litellm_core::responses::{route::Responses, types::ResponsesCall};
 use litellm_gateway_auth::AuthenticatedRequest;
 use litellm_host_http::Sse;
+use litellm_inference_responses::{route::Responses, types::ResponsesCall};
 use serde_json::json;
 
 use crate::{Error, Gateway, JsonObject, request};