Merge branch 'main' of https://github.com/BerriAI/litellm into include-cost-in-usage

.cargo/config.toml

@@ -17,3 +17,6 @@ rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
 
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[env]
+SQLX_OFFLINE = "true"

.circleci/config.yml

@@ -141,7 +141,7 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
     steps:
       - run:
           name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
@@ -167,9 +167,29 @@ commands:
             /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
             rustc --version
             cargo --version
+            { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
+  save_cargo_target:
+    steps:
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/debug
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -281,50 +301,11 @@ commands:
             # `uv sync --package litellm-enterprise` here — that overwrites the
             # shared .venv and strips out dev/test deps (pytest, prisma, etc.).
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
-  setup_litellm_test_deps:
+  install_windows_toolchain:
     steps:
-      - checkout
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
       - run:
-          name: Install Dependencies
+          name: Install Rust and uv
-          command: |
+          no_output_timeout: 30m
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
-
-jobs:
-  # Add Windows testing job
-  using_litellm_on_windows:
-    executor:
-      name: win/default
-      shell: powershell.exe
-    working_directory: ~/project
-    environment:
-      UV_PYTHON: "3.11"
-      CARGO_HTTP_MULTIPLEXING: "false"
-      CARGO_NET_RETRY: "5"
-    steps:
-      - checkout
-      - run:
-          name: Install Python
-          command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
-      - run:
-          name: Install Dependencies
-          environment:
-            UV_HTTP_TIMEOUT: "300"
           command: |
             $rustupInit = Join-Path $env:TEMP "rustup-init.exe"
             $rustupVersion = "1.28.2"
@@ -364,6 +345,55 @@ jobs:
             if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
+  setup_litellm_test_deps:
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
+
+jobs:
+  # Add Windows testing job
+  using_litellm_on_windows:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Install Dependencies
+          no_output_timeout: 30m
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             for ($attempt = 1; $attempt -le 5; $attempt++) {
               Write-Host "uv sync attempt $attempt/5"
               uv sync --frozen --group dev --python 3.11
@@ -379,16 +409,68 @@ jobs:
           name: Run Windows-specific test
           command: |
             uv run --no-sync python -m pytest tests/windows_tests/ -v
+
+  windows_release_wheel:
+    executor:
+      name: win/default
+      shell: powershell.exe
+      size: xlarge
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
       - run:
-          name: Guard against MAX_PATH-busting packaged wheel paths
+          name: Skip job when no windows-release-relevant files changed
+          shell: bash.exe
+          command: bash .circleci/scripts/path_filter.sh windows-release
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Record the Rust build environment for the release cargo cache key
+          command: |
+            & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: |
+            $fingerprints = "litellm-rust/target/release/.fingerprint"
+            if (Test-Path $fingerprints) {
+              Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
+            }
+      - run:
+          name: Build the release wheel and install it under a worst-case MAX_PATH prefix
+          no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
             $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
-            cargo --version
-            Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
             uv build --wheel --out-dir dist
-            uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
+            if ($LASTEXITCODE -ne 0) {
+              exit $LASTEXITCODE
+            }
+            python tests/windows_tests/check_windows_wheel_install.py
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/release
 
   base_sdk_install:
     docker:
@@ -404,6 +486,7 @@ jobs:
       - install_rust
       - run:
           name: Build the wheel
+          no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
@@ -416,6 +499,10 @@ jobs:
             uv venv /tmp/base-sdk --python 3.12
             VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
             /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+      - run:
+          name: Guard against MAX_PATH-busting packaged wheel paths
+          command: |
+            python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
 
   local_testing_part1:
     docker:
@@ -444,6 +531,7 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -3118,10 +3206,14 @@ jobs:
         type: enum
         enum: [standard, replica]
         default: standard
+      parallelism:
+        type: integer
+        default: 1
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: << parameters.parallelism >>
     steps:
       - setup_litellm_test_deps
       - when:
@@ -3247,6 +3339,7 @@ jobs:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: 4
     steps:
       - setup_litellm_test_deps
       - run:
@@ -3256,10 +3349,11 @@ jobs:
           name: Run unit tests
           command: |
             mkdir -p test-results/unit
-            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
+            shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
-            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
+            mapfile -t files < <(printf '%s\n' "${shard}")
             set +e
-            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
@@ -3326,23 +3420,17 @@ workflows:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, extensions, mcp, sdk, cost, browser]
+              suite: [management, accounting, database, providers, mcp, sdk, cost, security, browser]
-          filters:
+      - integration_contracts:
-            branches:
+          name: integration-extensions
-              only:
+          suite: extensions
-                - main
+          parallelism: 4
-                - /litellm_.*/
       - integration_contracts:
           name: integration-<< matrix.suite >>-replica
           matrix:
             parameters:
               suite: [management, database]
               mode: [replica]
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
   build_and_test:
     unless:
       or:
@@ -3350,101 +3438,61 @@ workflows:
         - not:
             equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
-      - using_litellm_on_windows:
+      - using_litellm_on_windows
-          filters: &main_branches
+      - windows_release_wheel
-            branches:
+      - unit
-              only:
-                - main
-                - /litellm_.*/
-      - unit:
-          filters: *main_branches
       - provider_replay_harness
-      - base_sdk_install:
+      - base_sdk_install
-          filters: *main_branches
+      - local_testing_part1
-      - local_testing_part1:
+      - local_testing_part2
-          filters: *main_branches
+      - langfuse_logging_unit_tests
-      - local_testing_part2:
+      - litellm_assistants_api_testing
-          filters: *main_branches
+      - litellm_router_testing
-      - langfuse_logging_unit_tests:
+      - litellm_router_unit_testing
-          filters: *main_branches
+      - auth_ui_unit_tests
-      - litellm_assistants_api_testing:
+      - build_docker_database_image
-          filters: *main_branches
+      - e2e_ui_testing
-      - litellm_router_testing:
+      - e2e_ui_testing_server_root_path
-          filters: *main_branches
-      - litellm_router_unit_testing:
-          filters: *main_branches
-      - auth_ui_unit_tests:
-          filters: *main_branches
-      - build_docker_database_image:
-          filters: *main_branches
-      - e2e_ui_testing:
-          filters: *main_branches
-      - e2e_ui_testing_server_root_path:
-          filters: *main_branches
       - build_and_test:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - e2e_openai_endpoints:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_logging_guardrails_model_info_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_spend_accuracy_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_multi_instance_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_store_model_in_db_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
+      - proxy_build_from_pip_tests
-      - proxy_build_from_pip_tests:
-          filters: *main_branches
       - proxy_pass_through_endpoint_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_e2e_anthropic_messages_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
+      - llm_translation_testing
-      - llm_translation_testing:
+      - realtime_translation_testing
-          filters: *main_branches
+      - agent_testing
-      - realtime_translation_testing:
+      - guardrails_testing
-          filters: *main_branches
+      - google_generate_content_endpoint_testing
-      - agent_testing:
+      - llm_responses_api_testing
-          filters: *main_branches
+      - ocr_testing
-      - guardrails_testing:
+      - search_testing
-          filters: *main_branches
+      - batches_testing
-      - google_generate_content_endpoint_testing:
+      - litellm_utils_testing
-          filters: *main_branches
+      - pass_through_unit_testing
-      - llm_responses_api_testing:
+      - image_gen_testing
-          filters: *main_branches
+      - logging_testing
-      - ocr_testing:
+      - audio_testing
-          filters: *main_branches
+      - redis_caching_unit_tests
-      - search_testing:
-          filters: *main_branches
-      - batches_testing:
-          filters: *main_branches
-      - litellm_utils_testing:
-          filters: *main_branches
-      - pass_through_unit_testing:
-          filters: *main_branches
-      - image_gen_testing:
-          filters: *main_branches
-      - logging_testing:
-          filters: *main_branches
-      - audio_testing:
-          filters: *main_branches
-      - redis_caching_unit_tests:
-          filters: *main_branches
       - upload-coverage:
           requires:
             - realtime_translation_testing
@@ -3469,18 +3517,12 @@ workflows:
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-          filters: *main_branches
+      - installing_litellm_on_python
-      - installing_litellm_on_python:
+      - installing_litellm_on_python_3_13
-          filters: *main_branches
+      - installing_litellm_on_python_v2_migration_resolver
-      - installing_litellm_on_python_3_13:
-          filters: *main_branches
-      - installing_litellm_on_python_v2_migration_resolver:
-          filters: *main_branches
       - helm_chart_testing:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - test_bad_database_url:
           requires:
             - build_docker_database_image
-          filters: *main_branches

.circleci/scripts/classify_changes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
 
 has_client=false
 has_backend=false
@@ -9,6 +9,7 @@ has_ci=false
 has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
+has_windows_release=false
 outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
@@ -22,6 +23,10 @@ while IFS= read -r file || [ -n "$file" ]; do
     tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
       has_provider_harness=true ;;
   esac
+  case "$file" in
+    litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
+      has_windows_release=true ;;
+  esac
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
@@ -46,6 +51,9 @@ case "$category" in
   provider-harness)
     [ "$has_provider_harness" = true ] && echo run || echo skip
     ;;
+  windows-release)
+    [ "$has_windows_release" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/run_integration.sh

@@ -26,6 +26,7 @@ guard_created=false
 guard_installed=false
 guard6_created=false
 guard6_installed=false
+egress_cgroup=litellm-integration
 cleanup() {
   original_status=$?
   trap - EXIT INT TERM
@@ -47,14 +48,14 @@ cleanup() {
     fi
   done
   if [ "$guard_installed" = true ]; then
-    sudo iptables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo iptables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard_created" = true ]; then
     sudo iptables -F integration_only || original_status=1
     sudo iptables -X integration_only || original_status=1
   fi
   if [ "$guard6_installed" = true ]; then
-    sudo ip6tables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo ip6tables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard6_created" = true ]; then
     sudo ip6tables -F integration_only || original_status=1
@@ -100,6 +101,8 @@ if [ "$mode" = parity ]; then
   export INTEGRATION_ROUTING=capture
 fi
 
+sudo mkdir -p "/sys/fs/cgroup/$egress_cgroup"
+echo "$$" | sudo tee "/sys/fs/cgroup/$egress_cgroup/cgroup.procs" > /dev/null
 sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
@@ -109,13 +112,13 @@ for service in postgres-db redis-cache; do
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
 sudo iptables -A integration_only -j REJECT
-sudo iptables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo iptables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard_installed=true
 sudo ip6tables -N integration_only
 guard6_created=true
 sudo ip6tables -A integration_only -o lo -j ACCEPT
 sudo ip6tables -A integration_only -j REJECT
-sudo ip6tables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo ip6tables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard6_installed=true
 
 if curl --noproxy '*' --connect-timeout 2 -s http://198.51.100.1 >/dev/null 2>&1; then
@@ -165,6 +168,7 @@ start_proxy() {
     "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
+    LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
     "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@@ -209,6 +213,15 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
+node_files=()
+if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
+  split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
+    | circleci tests split --split-by=timings --timings-type=filename)"
+  read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
+  test "${#node_files[@]}" -gt 0
+  printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
+fi
+
 env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
   DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
@@ -216,13 +229,14 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
+  LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
   INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
   INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
   INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
-  .venv/bin/python tests/integration/run.py "$suite" --results "$results"
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
 
 if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
   for covered_pid in "$proxy_pid" "$peer_pid"; do

.circleci/scripts/unit_selection.sh

@@ -5,6 +5,7 @@ flag="${1:?usage: unit_selection.sh <codecov flag>}"
 
 legacy_flags=(
   caching-local
+  core-utils
   enterprise-package
   enterprise-routing
   integrations
@@ -32,6 +33,7 @@ legacy_flags=(
 legacy_paths() {
   case "$1" in
     caching-local) echo tests/unit/caching ;;
+    core-utils) echo tests/unit/litellm_core_utils ;;
     enterprise-package)
       echo tests/unit/enterprise/integrations
       echo tests/unit/enterprise/proxy/auth
@@ -42,6 +44,9 @@ legacy_paths() {
       echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
     enterprise-routing)
       echo tests/unit/google_genai
+      echo tests/unit/router_strategy
+      echo tests/unit/router_utils
+      echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
       echo tests/unit/enterprise/enterprise_callbacks/send_emails
       echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
       echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
@@ -77,12 +82,14 @@ legacy_paths() {
       echo tests/unit/messages
       echo tests/unit/rag
       echo tests/unit/rerank_api
+      echo tests/unit/rust_bridge
       echo tests/unit/secret_managers
       echo tests/unit/vector_stores
       echo tests/unit/videos ;;
     proxy-db-auth-checks)
       echo tests/unit/proxy/auth/test_auth_checks.py
       echo tests/unit/proxy/auth/test_user_api_key_auth.py
+      echo tests/unit/proxy/test_credential_slot_registry.py
       echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
     proxy-db-budgets)
       echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
@@ -100,6 +107,7 @@ legacy_paths() {
       echo tests/unit/proxy/test_update_spend.py
       echo tests/unit/skills/test_skills_db.py ;;
     proxy-db-endpoints-and-responses)
+      echo tests/unit/proxy/lens
       echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
       echo tests/unit/proxy/common_utils/test_check_batch_cost.py
       echo tests/unit/proxy/common_utils/test_check_responses_cost.py
@@ -141,8 +149,13 @@ legacy_paths() {
       echo tests/unit/proxy/test_proxy_server.py ;;
     proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
     proxy-extras) echo tests/unit/litellm_proxy_extras ;;
-    proxy-infra) echo tests/unit/gateway ;;
+    proxy-infra)
-    responses-caching-types) echo tests/unit/types ;;
+      echo tests/unit/gateway
+      echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+      echo tests/unit/proxy/roi_calculator ;;
+    responses-caching-types)
+      find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
+      echo tests/unit/types ;;
     *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
   esac
 }

.circleci/tests.yml

@@ -369,6 +369,13 @@ workflows:
           reruns: 2
           base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
           pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-core-utils
+          flag: core-utils
+          shards: 2
+          reruns: 1
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
       - unit:
           name: unit-integrations
           flag: integrations

.githooks/commit-msg

@@ -42,7 +42,7 @@ case "$subject" in
         ;;
 esac
 
-ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert"
+ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert|security"
 # Description must not start with an uppercase letter — kept in sync with the
 # subjectPattern in .github/workflows/conventional-commits.yml so the local
 # hook is the strictly tighter of the two gates. (Without this guard, a commit
@@ -61,7 +61,7 @@ cat >&2 <<EOF
   Expected: <type>(<scope>)!: <description>
             (description must start with a lowercase letter)
 
-  Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert
+  Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert, security
   Examples:
     feat(router): add weighted round-robin strategy
     fix(bedrock): decouple STS region from aws_region_name

.github/CODEOWNERS

@@ -1,10 +1,2 @@
-/ui/ @yuneng-berri @ryan-crabbe-berri
-/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
-/ui/Dockerfile
-/ui/nginx.conf
-/ui/litellm-dashboard/src/lib/http/schema.d.ts
-/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
-/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
-/.github/CODEOWNERS @yuneng-berri

.github/actions/cache-cargo-build/action.yml

@@ -25,6 +25,7 @@ runs:
   using: composite
   steps:
     - name: Restore the Cargo registry and target directory
+      if: github.ref == 'refs/heads/main'
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         path: |
@@ -34,3 +35,15 @@ runs:
         key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
         restore-keys: |
           ${{ runner.os }}-maturin-${{ inputs.profile }}-
+
+    - name: Restore the Cargo registry and target directory
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          litellm-rust/target
+        key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-maturin-${{ inputs.profile }}-

.github/actions/cache-prisma-binaries/action.yml

@@ -30,6 +30,7 @@ runs:
         echo "version=${version}" >> "$GITHUB_OUTPUT"
 
     - name: Restore Prisma binaries
+      if: github.ref == 'refs/heads/main'
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         # ~/.cache/prisma-python holds the npm install tree prisma-client-py
@@ -38,3 +39,12 @@ runs:
           ~/.cache/prisma-python
           ~/.cache/prisma
         key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}
+
+    - name: Restore Prisma binaries
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cache/prisma-python
+          ~/.cache/prisma
+        key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}

.github/actions/cache-uv-downloads/action.yml

@@ -0,0 +1,25 @@
+name: "Cache uv downloads"
+description: >-
+  Restore the uv download cache on every run and save it only from main, so pull
+  requests reuse main's cache instead of evicting it with their own copies.
+
+runs:
+  using: composite
+  steps:
+    - name: Restore and save the uv download cache
+      if: github.ref == 'refs/heads/main'
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: ${{ env.UV_CACHE_DIR }}
+        key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
+
+    - name: Restore the uv download cache
+      if: github.ref != 'refs/heads/main'
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: ${{ env.UV_CACHE_DIR }}
+        key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-

.github/actions/setup-uv-with-retries/action.yml

@@ -17,6 +17,7 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}
 
     - name: Wait before attempt 2
       if: steps.attempt-1.outcome == 'failure'
@@ -30,6 +31,7 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}
 
     - name: Wait before attempt 3
       if: steps.attempt-2.outcome == 'failure'
@@ -41,3 +43,4 @@ runs:
       uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
+        save-cache: ${{ github.ref == 'refs/heads/main' }}

.github/ci-coverage-allowlist.yml

@@ -111,3 +111,10 @@ dockerfiles:
       An example image under cookbook/ that is documentation rather than a shipped artifact
     paths:
       - cookbook/litellm-ollama-docker-image/Dockerfile
+  - reason: >-
+      The Rust gateway image compiles the whole workspace in release mode, which is too slow for
+      a per-pull-request job while the gateway binary is still being assembled; the Rust lint,
+      clippy, and compile jobs already cover the code it packages. Revisit when the gateway is
+      published
+    paths:
+      - litellm-rust/crates/gateway/Dockerfile

.github/e2e-stack/select_tests.py

@@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
     r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
     r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
     r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
+    r"|^tests/e2e/logging/test_s3_log_e2e\.py$"
     r"|^tests/e2e/secret_manager/"
 )
 HARNESS: Final = re.compile(

.github/merge-smoke-tests.json

@@ -3,13 +3,13 @@
     "CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
     "CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
     "CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
-    "MODEL-ALLOW": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_allows_listed_model_for_key",
+    "MODEL-ALLOW": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_allows_listed_model_for_key",
-    "MODEL-DENY": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
+    "MODEL-DENY": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
     "COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
     "COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
-    "LOG-CONTENT-ON": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
+    "LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
-    "LOG-CONTENT-OFF": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
+    "LOG-CONTENT-OFF": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
-    "CALLBACK-SUCCESS": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
+    "CALLBACK-SUCCESS": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
-    "CALLBACK-FAILURE": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
+    "CALLBACK-FAILURE": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
   }
 }

.github/pull_request_template.md

@@ -54,7 +54,7 @@ After: the same request comes back with real token counts, so the dashboard show
 
 ## Affected release
 
-<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Drop the section otherwise -->
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1". Add the `backport-stable` label only when the regression is a P0, meaning its Linear ticket is Urgent (a security hole however narrow, data loss, or a crash or outage for every user on that version), because every labeled PR must be cherry-picked onto the baking rc line before the stable can be tagged; every other regression fix ships in the next rc unlabeled. Drop the section otherwise -->
 
 ## Linear ticket
 
@@ -65,7 +65,7 @@ After: the same request comes back with real token counts, so the dashboard show
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
 - [ ] I have added meaningful tests
-- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
+- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
 - [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
 - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)

.github/scripts/assert_ci_coverage.py

@@ -516,7 +516,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         str(path.relative_to(repo_root))
         for folders in groups.values()
         for folder in folders
-        for path in (integration_root / folder).glob("test_*.py")
+        for path in (integration_root / folder).rglob("test_*.py")
     )
     browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
     browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()

.github/scripts/verify_linux_native_wheel.py

@@ -214,7 +214,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 40_000_000
+    native_size_limit: Final = 45_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),

.github/workflows/_test-unit-base.yml

@@ -132,12 +132,7 @@ jobs:
       - name: Cache uv dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 5
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        uses: ./.github/actions/cache-uv-downloads
-        with:
-          path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
 
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
@@ -274,7 +269,7 @@ jobs:
       - name: Upload to Codecov
         id: codecov-upload
         continue-on-error: true
-        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
         with:
           use_oidc: true
           directory: coverage-reports
@@ -285,7 +280,7 @@ jobs:
       - name: Upload to Codecov (retry)
         if: steps.codecov-upload.outcome == 'failure'
         continue-on-error: true
-        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
         with:
           use_oidc: true
           directory: coverage-reports

.github/workflows/conventional-commits.yml

@@ -41,6 +41,7 @@ jobs:
             ci
             chore
             revert
+            security
           requireScope: false
           subjectPattern: ^(?![A-Z]).+$
           subjectPatternError: |

.github/workflows/create-rc-branch.yml

@@ -15,6 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+    outputs:
+      version: ${{ steps.version.outputs.version }}
     steps:
       - name: Require main
         env:
@@ -64,3 +66,14 @@ jobs:
               sha: context.sha,
             });
             core.info(`Created branch ${branchName} at ${context.sha}`);
+
+  linear-release:
+    name: Move the Linear release to rc
+    needs: create-rc-branch
+    permissions:
+      contents: read
+    uses: ./.github/workflows/linear-release.yml
+    with:
+      rc_version: ${{ needs.create-rc-branch.outputs.version }}
+    secrets:
+      LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}

.github/workflows/lens-worker.yml

@@ -0,0 +1,66 @@
+name: Lens Worker Image
+
+on:
+  pull_request:
+    branches: [main, litellm_oss_branch, "litellm_**"]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/lens/**
+      - .github/workflows/lens-worker.yml
+  push:
+    branches: [main]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/lens/**
+      - .github/workflows/lens-worker.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lens-worker-image:
+    permissions:
+      contents: read
+      packages: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Build Lens worker
+        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Verify standalone imports with a read-only filesystem
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
+            --security-opt no-new-privileges --entrypoint python \
+            lens-worker:${{ github.sha }} -c '
+          import os
+          import lens.worker
+          from lens.trace_store import trace_store
+          assert os.getuid() == 65532
+          with trace_store() as store:
+              assert store.count() == 0
+          '
+      - name: Verify recovery after temporary storage fills
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
+            -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
+            --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
+      - name: Publish versioned Lens worker
+        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+        env:
+          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }}
+        run: |
+          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
+          docker tag lens-worker:${{ github.sha }} "$IMAGE"
+          docker push "$IMAGE"
+          printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/linear-release.yml

@@ -0,0 +1,131 @@
+name: Linear Release
+
+on:
+  push:
+    branches:
+      - main
+      - "rc/**"
+  release:
+    types: [published]
+  workflow_call:
+    inputs:
+      rc_version:
+        description: "X.Y.0 release whose rc branch was just cut"
+        required: true
+        type: string
+    secrets:
+      LINEAR_API_KEY:
+        required: true
+
+permissions: {}
+
+jobs:
+  linear-release:
+    name: Linear Release
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Plan
+        id: plan
+        env:
+          EVENT: ${{ github.event_name }}
+          REF_NAME: ${{ github.ref_name }}
+          BEFORE: ${{ github.event.before }}
+          CREATED: ${{ github.event.created }}
+          RC_VERSION: ${{ inputs.rc_version }}
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+          PRERELEASE: ${{ github.event.release.prerelease }}
+        run: |
+          set -euo pipefail
+          sync_base="${BEFORE}"
+          if [ "${CREATED}" = "true" ]; then
+            sync_base=""
+          fi
+          if [ -n "${RC_VERSION}" ]; then
+            echo "version=${RC_VERSION}" >> "$GITHUB_OUTPUT"
+            echo "stage=rc" >> "$GITHUB_OUTPUT"
+          elif [ "${EVENT}" = "release" ]; then
+            if [ "${PRERELEASE}" = "true" ] || ! echo "${RELEASE_TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.0$'; then
+              echo "::notice::${RELEASE_TAG} is not an X.Y.0 stable release; nothing to complete"
+              exit 0
+            fi
+            echo "version=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT"
+            echo "complete=true" >> "$GITHUB_OUTPUT"
+          elif [ "${REF_NAME}" = "main" ]; then
+            version="$(python3 .github/scripts/read_rc_version.py | cut -d= -f2)"
+            status=0
+            git ls-remote --exit-code --heads origin "rc/${version}" > /dev/null || status=$?
+            case "${status}" in
+              0)
+                IFS=. read -r major minor _ <<< "${version}"
+                version="${major}.$((minor + 1)).0"
+                ;;
+              2) ;;
+              *)
+                echo "::error::could not check whether rc/${version} exists (git ls-remote exit ${status})"
+                exit 1
+                ;;
+            esac
+            echo "version=${version}" >> "$GITHUB_OUTPUT"
+            echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT"
+            echo "main=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=${REF_NAME#rc/}" >> "$GITHUB_OUTPUT"
+            echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT"
+            echo "stage=rc" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Sync commits into the release
+        if: steps.plan.outputs.sync_base != ''
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: sync
+          name: LiteLLM ${{ steps.plan.outputs.version }}
+          version: ${{ steps.plan.outputs.version }}
+          base_ref: ${{ steps.plan.outputs.sync_base }}
+          cli_version: v0.18.0
+
+      - name: Keep the main stage unless the rc branch was cut during this run
+        id: main_stage
+        if: steps.plan.outputs.main == 'true'
+        env:
+          VERSION: ${{ steps.plan.outputs.version }}
+        run: |
+          set -euo pipefail
+          status=0
+          git ls-remote --exit-code --heads origin "rc/${VERSION}" > /dev/null || status=$?
+          case "${status}" in
+            0) echo "::notice::rc/${VERSION} was cut during this run; leaving the release in its rc stage" ;;
+            2) echo "stage=main" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::could not check whether rc/${VERSION} exists (git ls-remote exit ${status})"
+              exit 1
+              ;;
+          esac
+
+      - name: Move the release to its stage
+        if: steps.plan.outputs.stage != '' || steps.main_stage.outputs.stage != ''
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: update
+          stage: ${{ steps.plan.outputs.stage || steps.main_stage.outputs.stage }}
+          version: ${{ steps.plan.outputs.version }}
+          cli_version: v0.18.0
+
+      - name: Complete the release
+        if: steps.plan.outputs.complete == 'true'
+        uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0
+        with:
+          access_key: ${{ secrets.LINEAR_API_KEY }}
+          command: complete
+          version: ${{ steps.plan.outputs.version }}
+          cli_version: v0.18.0

.github/workflows/mutation-test.yml

@@ -44,6 +44,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -53,6 +54,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 

.github/workflows/test-code-quality.yml

@@ -44,6 +44,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -53,6 +54,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 
@@ -80,6 +92,11 @@ jobs:
       - name: test_e2e_changed_gate
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
 
+      - name: test_e2e_metadata
+        env:
+          PYTHONPATH: tests/e2e
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
+
       - name: Check merge smoke harness
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
 
@@ -146,6 +163,9 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_unbounded_in_lists (fails on findings not in the baseline)
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-postgres.yml

@@ -24,6 +24,7 @@ jobs:
     timeout-minutes: ${{ matrix.job-timeout-minutes }}
     permissions:
       contents: read
+      id-token: write
 
     services:
       postgres:
@@ -94,7 +95,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip'
+        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
         timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
@@ -105,6 +106,18 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-postgres-
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
+        timeout-minutes: 5
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 12
@@ -134,9 +147,21 @@ jobs:
         env:
           TEST_PATH: ${{ matrix.test-path }}
           WORKERS: ${{ matrix.workers }}
+          PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
           else
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
           fi
+
+      - name: Upload Lens database coverage
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          use_oidc: true
+          version: v11.3.1
+          root_dir: ${{ github.workspace }}
+          files: coverage-lens-postgres.xml
+          flags: lens-postgres
+          fail_ci_if_error: true

.github/workflows/test-redis-compat.yml

@@ -12,9 +12,9 @@ on:
       - "litellm/caching/evicted_client_closer.py"
       - "tests/unit/test_redis.py"
       - "tests/local_testing/test_caching.py"
-      - "tests/test_litellm/caching/test_redis_connection_pool.py"
+      - "tests/unit/caching/test_redis_connection_pool.py"
-      - "tests/test_litellm/caching/test_redis_cluster_cache.py"
+      - "tests/unit/caching/test_redis_cluster_cache.py"
-      - "tests/test_litellm/caching/test_evicted_client_closer.py"
+      - "tests/unit/caching/test_evicted_client_closer.py"
       - ".github/workflows/test-redis-compat.yml"
       - "pyproject.toml"
       - "uv.lock"
@@ -85,9 +85,9 @@ jobs:
           redis-server --version
           uv run --no-sync pytest \
             tests/unit/test_redis.py \
-            tests/test_litellm/caching/test_redis_connection_pool.py \
+            tests/unit/caching/test_redis_connection_pool.py \
-            tests/test_litellm/caching/test_redis_cluster_cache.py \
+            tests/unit/caching/test_redis_cluster_cache.py \
-            tests/test_litellm/caching/test_evicted_client_closer.py \
+            tests/unit/caching/test_evicted_client_closer.py \
             tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
             tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
             --tb=short -vv \
@@ -98,7 +98,7 @@ jobs:
 
       - name: Upload Redis coverage
         if: matrix.redis-version == '5.3.1'
-        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
         with:
           use_oidc: true
           files: coverage-redis.xml

.github/workflows/test-rust.yml

@@ -14,7 +14,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -24,7 +23,7 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
   pull_request:
     branches:
@@ -42,7 +41,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -52,7 +50,7 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
 
 permissions:
@@ -85,6 +83,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
@@ -123,6 +122,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: cargo nextest run --workspace --locked
 
@@ -164,6 +164,7 @@ jobs:
         with:
           workspaces: litellm-rust
           cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - run: uv build --wheel --out-dir dist
 
@@ -171,7 +172,7 @@ jobs:
         env:
           RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
-      - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+      - run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
 
       - name: Run pytest tests/test_litellm_rust with the compiled extension
         run: make test-rust-extension

.github/workflows/test-terraform-provider.yml

@@ -77,6 +77,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -86,6 +87,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
 

.github/workflows/test-unit-documentation.yml

@@ -54,7 +54,7 @@ jobs:
           version: "0.10.9"
 
       - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip'
+        if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -64,6 +64,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/cache-cargo-build

.github/workflows/test-unit.yml

@@ -61,7 +61,8 @@ jobs:
 
           - shard: core-utils
             artifact-name: core-utils
-            test-path: "tests/test_litellm/litellm_core_utils"
+            test-path: ""
+            unit-flag: core-utils
             workers: 2
             reruns: 1
             timeout-minutes: 20
@@ -69,9 +70,7 @@ jobs:
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
-            test-path: >-
+            test-path: ""
-              tests/test_litellm/router_utils
-              tests/test_litellm/router_strategy
             unit-flag: enterprise-routing
             workers: 2
             reruns: 2
@@ -80,7 +79,9 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: ""
+            test-path: >-
+              tests/test_litellm/integrations
+              tests/test_litellm/tracing
             unit-flag: integrations
             workers: 2
             reruns: 3
@@ -89,7 +90,7 @@ jobs:
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
-            test-path: "tests/test_litellm/llms/vertex_ai"
+            test-path: ""
             unit-flag: llm-vertex-ai
             workers: 1
             reruns: 2
@@ -98,7 +99,7 @@ jobs:
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
-            test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            test-path: ""
             unit-flag: llm-other-providers
             workers: 2
             reruns: 2
@@ -108,10 +109,6 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/test_litellm/interactions
-              tests/test_litellm/ocr
-              tests/test_litellm/passthrough
-              tests/test_litellm/rust_bridge
               tests/test_litellm/test_*.py
             unit-flag: misc
             workers: 2
@@ -122,10 +119,19 @@ jobs:
           - shard: proxy-auth
             artifact-name: proxy-auth
             test-path: >-
-              tests/test_litellm/proxy/auth
+              tests/unit/proxy/auth
-              tests/test_litellm/proxy/hooks
+              tests/unit/proxy/hooks
-              tests/test_litellm/proxy/policy_engine
+              tests/unit/proxy/policy_engine
-              tests/test_litellm/proxy/client
+              tests/unit/proxy/client
+              --ignore=tests/unit/proxy/auth/test_auth_checks.py
+              --ignore=tests/unit/proxy/auth/test_user_api_key_auth.py
+              --ignore=tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+              --ignore=tests/unit/proxy/auth/test_jwt.py
+              --ignore=tests/unit/proxy/auth/test_models_fallback_endpoint.py
+              --ignore=tests/unit/proxy/auth/test_multipart_bypass_repro.py
+              --ignore=tests/unit/proxy/auth/test_proxy_routes.py
+              --ignore=tests/unit/proxy/hooks/test_banned_keyword_list.py
+              --ignore=tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -134,38 +140,46 @@ jobs:
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
             test-path: >-
-              tests/test_litellm/proxy/analytics_endpoints
+              tests/unit/proxy/analytics_endpoints
-              tests/test_litellm/proxy/management_endpoints
+              tests/unit/proxy/management_endpoints
-              tests/test_litellm/proxy/list_api
+              tests/unit/proxy/list_api
-              tests/test_litellm/proxy/memory
+              tests/unit/proxy/memory
-              tests/test_litellm/proxy/guardrails
+              tests/unit/proxy/guardrails
-              tests/test_litellm/proxy/management_helpers
+              tests/unit/proxy/management_helpers
-              tests/test_litellm/proxy/anthropic_endpoints
+              --ignore=tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
-              tests/test_litellm/proxy/google_endpoints
+              --ignore=tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
-              tests/test_litellm/proxy/openai_files_endpoint
+              --ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
-              tests/test_litellm/proxy/batches_endpoints
+              --ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
-              tests/test_litellm/proxy/container_endpoints
+              --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
-              tests/test_litellm/proxy/fine_tuning_endpoints
+              --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
-              tests/test_litellm/proxy/vector_store_files_endpoints
+              --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
-              tests/test_litellm/proxy/video_endpoints
+              --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
-              tests/test_litellm/proxy/response_api_endpoints
+              tests/unit/proxy/anthropic_endpoints
-              tests/test_litellm/proxy/image_endpoints
+              tests/unit/proxy/google_endpoints
-              tests/test_litellm/proxy/ocr_endpoints
+              tests/unit/proxy/openai_files_endpoint
-              tests/test_litellm/proxy/vector_store_endpoints
+              tests/unit/proxy/batches_endpoints
-              tests/test_litellm/proxy/agent_endpoints
+              tests/unit/proxy/container_endpoints
-              tests/test_litellm/proxy/a2a
+              tests/unit/proxy/fine_tuning_endpoints
-              tests/test_litellm/proxy/credential_endpoints
+              tests/unit/proxy/vector_store_files_endpoints
-              tests/test_litellm/proxy/discovery_endpoints
+              tests/unit/proxy/video_endpoints
-              tests/test_litellm/proxy/health_endpoints
+              tests/unit/proxy/response_api_endpoints
-              tests/test_litellm/proxy/shutdown
+              tests/unit/proxy/image_endpoints
-              tests/test_litellm/proxy/public_endpoints
+              tests/unit/proxy/ocr_endpoints
-              tests/test_litellm/proxy/prompts
+              tests/unit/proxy/vector_store_endpoints
-              tests/test_litellm/proxy/rag_endpoints
+              tests/unit/proxy/agent_endpoints
-              tests/test_litellm/proxy/rerank_endpoints
+              tests/unit/proxy/a2a
-              tests/test_litellm/proxy/realtime_endpoints
+              tests/unit/proxy/credential_endpoints
-              tests/test_litellm/proxy/ui_crud_endpoints
+              tests/unit/proxy/discovery_endpoints
-              tests/test_litellm/proxy/config_resolvers
+              tests/unit/proxy/health_endpoints
-              tests/test_litellm/proxy/utils
+              tests/unit/proxy/shutdown
+              tests/unit/proxy/public_endpoints
+              tests/unit/proxy/prompts
+              tests/unit/proxy/rag_endpoints
+              tests/unit/proxy/rerank_endpoints
+              tests/unit/proxy/realtime_endpoints
+              tests/unit/proxy/ui_crud_endpoints
+              tests/unit/proxy/config_resolvers
+              tests/unit/proxy/utils
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -173,7 +187,7 @@ jobs:
 
           - shard: proxy-server
             artifact-name: proxy-server
-            test-path: "tests/test_litellm/proxy/proxy_server"
+            test-path: "tests/unit/proxy/proxy_server"
             workers: 4
             reruns: 2
             timeout-minutes: 60
@@ -182,17 +196,108 @@ jobs:
           - shard: proxy-infra
             artifact-name: proxy-infra
             test-path: >-
-              tests/test_litellm/proxy/db
+              tests/unit/proxy/db
-              tests/test_litellm/proxy/middleware
+              --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
-              tests/test_litellm/proxy/spend_tracking
+              --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py
+              tests/unit/proxy/middleware
+              --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py
+              tests/unit/proxy/spend_tracking
+              --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py
+              tests/unit/proxy/pass_through_endpoints
               tests/test_litellm/proxy/pass_through_endpoints
-              tests/test_litellm/proxy/_experimental
+              tests/unit/proxy/_experimental
-              tests/test_litellm/proxy/experimental
+              --ignore=tests/unit/proxy/_experimental/mcp_server
-              tests/test_litellm/proxy/common_utils
+              tests/unit/proxy/experimental
-              tests/test_litellm/proxy/enterprise_billing
+              tests/unit/proxy/common_utils
-              tests/test_litellm/proxy/types_utils
+              --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py
-              tests/test_litellm/proxy/logging_endpoints
+              --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py
-              tests/test_litellm/proxy/test_*.py
+              --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py
+              --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
+              --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py
+              tests/unit/proxy/enterprise_billing
+              tests/unit/proxy/types_utils
+              tests/unit/proxy/logging_endpoints
+              tests/unit/proxy/test__types.py
+              tests/unit/proxy/test_aiohttp_cleanup_closed.py
+              tests/unit/proxy/test_aiohttp_session_recovery.py
+              tests/unit/proxy/test_api_key_masking_in_errors.py
+              tests/unit/proxy/test_audio_speech_prometheus_hooks.py
+              tests/unit/proxy/test_batch_expiry.py
+              tests/unit/proxy/test_batch_metadata_none_fix.py
+              tests/unit/proxy/test_batch_retrieve_bedrock.py
+              tests/unit/proxy/test_batch_x_litellm_model_encoding.py
+              tests/unit/proxy/test_blocked_response_usage.py
+              tests/unit/proxy/test_body_snapshot_callback_params.py
+              tests/unit/proxy/test_budget_reservation.py
+              tests/unit/proxy/test_bug_report_config.py
+              tests/unit/proxy/test_caching_routes.py
+              tests/unit/proxy/test_chat_completion_metadata.py
+              tests/unit/proxy/test_claude_code_marketplace.py
+              tests/unit/proxy/test_collector.py
+              tests/unit/proxy/test_common_request_processing.py
+              tests/unit/proxy/test_component_allowlists.py
+              tests/unit/proxy/test_conftest.py
+              tests/unit/proxy/test_cors_config.py
+              tests/unit/proxy/test_custom_proxy.py
+              tests/unit/proxy/test_dynamic_mcp_route.py
+              tests/unit/proxy/test_empty_model_list.py
+              tests/unit/proxy/test_enforce_user_param.py
+              tests/unit/proxy/test_fallback_management_endpoints.py
+              tests/unit/proxy/test_fastapi_offline_routes.py
+              tests/unit/proxy/test_filter_models_by_team_access_group.py
+              tests/unit/proxy/test_health_check_functions.py
+              tests/unit/proxy/test_health_check_max_tokens.py
+              tests/unit/proxy/test_init_litellm_callbacks.py
+              tests/unit/proxy/test_langfuse_passthrough_security.py
+              tests/unit/proxy/test_lazy_openapi_snapshot.py
+              tests/unit/proxy/test_litellm_pre_call_utils.py
+              tests/unit/proxy/test_max_budget_env_var.py
+              tests/unit/proxy/test_mcp_asgi_response.py
+              tests/unit/proxy/test_model_based_routing_files_batches.py
+              tests/unit/proxy/test_model_deprecations_endpoint.py
+              tests/unit/proxy/test_model_dump_with_preserved_fields.py
+              tests/unit/proxy/test_model_id_header_propagation.py
+              tests/unit/proxy/test_model_info_default_limits.py
+              tests/unit/proxy/test_model_level_guardrails.py
+              tests/unit/proxy/test_model_list_aliases.py
+              tests/unit/proxy/test_model_list_callback_filter.py
+              tests/unit/proxy/test_model_list_discoverable.py
+              tests/unit/proxy/test_model_list_healthy_only.py
+              tests/unit/proxy/test_modify_response_streaming_passthrough.py
+              tests/unit/proxy/test_native_compaction.py
+              tests/unit/proxy/test_openai_ws_passthrough_routes.py
+              tests/unit/proxy/test_openapi_schema_validation.py
+              tests/unit/proxy/test_plugin_routes.py
+              tests/unit/proxy/test_pointfive_dashboard_config.py
+              tests/unit/proxy/test_pointfive_ui_callback.py
+              tests/unit/proxy/test_pricing_field_strip.py
+              tests/unit/proxy/test_prisma_engine_watchdog.py
+              tests/unit/proxy/test_prisma_migration.py
+              tests/unit/proxy/test_prometheus_cleanup.py
+              tests/unit/proxy/test_prometheus_metrics_server.py
+              tests/unit/proxy/test_provider_url_destination_guard.py
+              tests/unit/proxy/test_proxy_cli.py
+              tests/unit/proxy/test_proxy_logging_hook_detection.py
+              tests/unit/proxy/test_proxy_types.py
+              tests/unit/proxy/test_pyroscope.py
+              tests/unit/proxy/test_read_model_list.py
+              tests/unit/proxy/test_redis_auth_cache_flag.py
+              tests/unit/proxy/test_response_model_sanitization.py
+              tests/unit/proxy/test_route_a2a_models.py
+              tests/unit/proxy/test_route_llm_request.py
+              tests/unit/proxy/test_route_priority.py
+              tests/unit/proxy/test_sensitive_route_auth.py
+              tests/unit/proxy/test_shared_health_check.py
+              tests/unit/proxy/test_spend_log_cleanup.py
+              tests/unit/proxy/test_swagger_chat_completions.py
+              tests/unit/proxy/test_team_member_update.py
+              tests/unit/proxy/test_team_org_move.py
+              tests/unit/proxy/test_tools_allowlist_enforcement.py
+              tests/unit/proxy/test_tracing_endpoints.py
+              tests/unit/proxy/test_update_llm_router_resilience.py
+              tests/unit/proxy/test_zerobus_dashboard_config.py
+              tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
+              tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
             unit-flag: proxy-infra
             workers: 4
             reruns: 2
@@ -228,9 +333,7 @@ jobs:
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
-            test-path: >-
+            test-path: ""
-              tests/test_litellm/responses
-              tests/test_litellm/caching
             unit-flag: responses-caching-types
             workers: 2
             reruns: 2

.gitignore

@@ -104,7 +104,7 @@ litellm_config.yaml
 .cursor
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
-tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
 scripts/test_vertex_ai_search.py
 LAZY_LOADING_IMPROVEMENTS.md
 STABILIZATION_TODO.md

AGENTS.md

@@ -27,7 +27,7 @@ Never test structure of code only function of it
 
 A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
 
-`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
+`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
 

ARCHITECTURE.md

@@ -255,7 +255,7 @@ Conventions to follow when touching this layer:
 | Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. |
 | Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. |
 
-To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`.
+To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/unit/repositories/`.
 
 ---
 
@@ -336,7 +336,7 @@ Each translation is isolated in its own file, making it easy to test and modify
 | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
 | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
 | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
-| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
 | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
 | `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
 | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |

CONTRIBUTING.md

@@ -14,7 +14,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
+  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
 
 #### UI PRs
 
@@ -72,7 +72,7 @@ make format
 make lint
 
 # Run the tests covering your change (CI runs the full suite)
-uv run pytest tests/test_litellm/<your_test_file>.py -v
+uv run pytest tests/unit/<your_test_file>.py -v
 
 # Commit your changes (must follow Conventional Commits — see above)
 git add .
@@ -88,7 +88,7 @@ git push origin feature/your-feature
 
 ### Where to Add Tests
 
-Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+Add your tests to the [`tests/unit/` directory](https://github.com/BerriAI/litellm/tree/main/tests/unit).
 
 - This directory mirrors the structure of the `litellm/` directory
 - **Only add mocked tests** - no real LLM API calls in this directory
@@ -96,10 +96,10 @@ Add your tests to the [`tests/test_litellm/` directory](https://github.com/Berri
 
 ### File Naming Convention
 
-The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+The `tests/unit/` directory follows the same structure as `litellm/`:
 
-- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
+- `litellm/proxy/caching_routes.py` → `tests/unit/proxy/test_caching_routes.py`
-- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+- `litellm/utils.py` → `tests/unit/test_utils.py`
 
 ### Example Test
 
@@ -125,10 +125,10 @@ def test_your_feature():
 
 Run the tests covering your change:
 ```bash
-uv run pytest tests/test_litellm/test_your_file.py -v
+uv run pytest tests/unit/test_your_file.py -v
 ```
 
-`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
+`tests/unit` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
 
 If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
 

Makefile

@@ -4,7 +4,7 @@
 .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	test-rust-extension \
+	test-rust-extension rust-sqlx-prepare \
 	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
@@ -42,7 +42,7 @@ help:
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit          - Run unit tests (tests/unit and tests/test_litellm)"
 	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
 	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
 	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
@@ -56,6 +56,7 @@ help:
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
+	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
 	@echo ""
 	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
 	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
@@ -301,38 +302,122 @@ test-rust-extension:
 	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
 	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
 	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
-		--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
+		--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
 		litellm.rust_bridge._native && \
 	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
 	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
 
+rust-sqlx-prepare:
+	cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
+
 test: install-test-deps
 	$(UV_RUN) pytest tests/
 
 test-unit: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
+	$(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4
 
 # Matrix test targets (matching CI workflow groups)
 test-unit-llms: install-test-deps
 	$(UV_RUN) pytest tests/unit/llms --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-guardrails: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/guardrails tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-core: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20
+
+PROXY_INFRA_ROOT_TESTS := \
+	tests/unit/proxy/test__types.py \
+	tests/unit/proxy/test_aiohttp_cleanup_closed.py \
+	tests/unit/proxy/test_aiohttp_session_recovery.py \
+	tests/unit/proxy/test_api_key_masking_in_errors.py \
+	tests/unit/proxy/test_audio_speech_prometheus_hooks.py \
+	tests/unit/proxy/test_batch_expiry.py \
+	tests/unit/proxy/test_batch_metadata_none_fix.py \
+	tests/unit/proxy/test_batch_retrieve_bedrock.py \
+	tests/unit/proxy/test_batch_x_litellm_model_encoding.py \
+	tests/unit/proxy/test_blocked_response_usage.py \
+	tests/unit/proxy/test_body_snapshot_callback_params.py \
+	tests/unit/proxy/test_budget_reservation.py \
+	tests/unit/proxy/test_bug_report_config.py \
+	tests/unit/proxy/test_caching_routes.py \
+	tests/unit/proxy/test_chat_completion_metadata.py \
+	tests/unit/proxy/test_claude_code_marketplace.py \
+	tests/unit/proxy/test_collector.py \
+	tests/unit/proxy/test_common_request_processing.py \
+	tests/unit/proxy/test_component_allowlists.py \
+	tests/unit/proxy/test_conftest.py \
+	tests/unit/proxy/test_cors_config.py \
+	tests/unit/proxy/test_custom_proxy.py \
+	tests/unit/proxy/test_dynamic_mcp_route.py \
+	tests/unit/proxy/test_empty_model_list.py \
+	tests/unit/proxy/test_enforce_user_param.py \
+	tests/unit/proxy/test_fallback_management_endpoints.py \
+	tests/unit/proxy/test_fastapi_offline_routes.py \
+	tests/unit/proxy/test_filter_models_by_team_access_group.py \
+	tests/unit/proxy/test_health_check_functions.py \
+	tests/unit/proxy/test_health_check_max_tokens.py \
+	tests/unit/proxy/test_init_litellm_callbacks.py \
+	tests/unit/proxy/test_langfuse_passthrough_security.py \
+	tests/unit/proxy/test_lazy_openapi_snapshot.py \
+	tests/unit/proxy/test_litellm_pre_call_utils.py \
+	tests/unit/proxy/test_max_budget_env_var.py \
+	tests/unit/proxy/test_mcp_asgi_response.py \
+	tests/unit/proxy/test_model_based_routing_files_batches.py \
+	tests/unit/proxy/test_model_deprecations_endpoint.py \
+	tests/unit/proxy/test_model_dump_with_preserved_fields.py \
+	tests/unit/proxy/test_model_id_header_propagation.py \
+	tests/unit/proxy/test_model_info_default_limits.py \
+	tests/unit/proxy/test_model_level_guardrails.py \
+	tests/unit/proxy/test_model_list_aliases.py \
+	tests/unit/proxy/test_model_list_callback_filter.py \
+	tests/unit/proxy/test_model_list_discoverable.py \
+	tests/unit/proxy/test_model_list_healthy_only.py \
+	tests/unit/proxy/test_modify_response_streaming_passthrough.py \
+	tests/unit/proxy/test_native_compaction.py \
+	tests/unit/proxy/test_openai_ws_passthrough_routes.py \
+	tests/unit/proxy/test_openapi_schema_validation.py \
+	tests/unit/proxy/test_plugin_routes.py \
+	tests/unit/proxy/test_pointfive_dashboard_config.py \
+	tests/unit/proxy/test_pointfive_ui_callback.py \
+	tests/unit/proxy/test_pricing_field_strip.py \
+	tests/unit/proxy/test_prisma_engine_watchdog.py \
+	tests/unit/proxy/test_prisma_migration.py \
+	tests/unit/proxy/test_prometheus_cleanup.py \
+	tests/unit/proxy/test_prometheus_metrics_server.py \
+	tests/unit/proxy/test_provider_url_destination_guard.py \
+	tests/unit/proxy/test_proxy_cli.py \
+	tests/unit/proxy/test_proxy_logging_hook_detection.py \
+	tests/unit/proxy/test_proxy_types.py \
+	tests/unit/proxy/test_pyroscope.py \
+	tests/unit/proxy/test_read_model_list.py \
+	tests/unit/proxy/test_redis_auth_cache_flag.py \
+	tests/unit/proxy/test_response_model_sanitization.py \
+	tests/unit/proxy/test_route_a2a_models.py \
+	tests/unit/proxy/test_route_llm_request.py \
+	tests/unit/proxy/test_route_priority.py \
+	tests/unit/proxy/test_sensitive_route_auth.py \
+	tests/unit/proxy/test_shared_health_check.py \
+	tests/unit/proxy/test_spend_log_cleanup.py \
+	tests/unit/proxy/test_swagger_chat_completions.py \
+	tests/unit/proxy/test_team_member_update.py \
+	tests/unit/proxy/test_team_org_move.py \
+	tests/unit/proxy/test_tools_allowlist_enforcement.py \
+	tests/unit/proxy/test_tracing_endpoints.py \
+	tests/unit/proxy/test_update_llm_router_resilience.py \
+	tests/unit/proxy/test_zerobus_dashboard_config.py
 
 test-unit-proxy-misc: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/test_litellm/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints $(PROXY_INFRA_ROOT_TESTS) tests/unit/proxy/test_proxy_server_endpoints_and_startup.py tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
 	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20
 
 test-unit-core-utils: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20
 
 test-unit-other: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/test_litellm/anthropic_interface tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
 	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20

README.md

@@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/routes/allowlist.py

@@ -81,6 +81,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Spend / analytics
     "/spend/",
     "/analytics/",
+    "/lens/",
+    "/v1/traces",
     "/global/",
     "/user_agent",
     "/usage/",
@@ -144,6 +146,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
     {
         "/",
         "/routes",
+        "/lens",
         "/openapi.json",
         "/docs",
         "/docs/oauth2-redirect",

basedpyright-code-budget.json

@@ -99,7 +99,7 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44358
+    "limit": 44802
   },
   "reportUnknownLambdaType": {
     "limit": 109

cookbook/misc/config.yaml

@@ -24,7 +24,7 @@ model_list:
   - model_name: sagemaker-completion-model
     litellm_params:
       model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4
-      input_cost_per_second: 0.000420  
+      cost_per_second: 0.000420
   - model_name: text-embedding-ada-002
     litellm_params: 
       model: azure/azure-embedding-model

cookbook/misc/test_responses_api.py

@@ -12,7 +12,7 @@ def encode_image(image_path):
 
 
 # Path to your image
-image_path = "litellm/proxy/logo.jpg"
+image_path = "litellm/proxy/logo.png"
 
 # Getting the Base64 string
 base64_image = encode_image(image_path)
@@ -27,7 +27,7 @@ response = client.responses.create(
                 {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
-                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                    "image_url": f"data:image/png;base64,{base64_image}",
                 },
             ],
         }

deploy/lens/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
+COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
+USER 65532:65532
+CMD ["python", "-m", "lens.worker"]

deploy/lens/Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+**
+!litellm/
+!litellm/proxy/
+!litellm/proxy/lens/
+!litellm/proxy/lens/__init__.py
+!litellm/proxy/lens/models.py
+!litellm/proxy/lens/analysis.py
+!litellm/proxy/lens/worker.py

deploy/lens/README.md

@@ -0,0 +1,117 @@
+# Lens worker
+
+Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
+
+## Start a worker
+
+Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
+
+In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in
+
+The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
+
+The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds
+
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+
+```bash
+docker compose --env-file /path/to/lens.env -f compose.yaml up -d
+```
+
+Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
+
+The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
+
+The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
+
+If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
+
+V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
+
+## Configure a lens
+
+Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute
+
+Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage
+
+Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
+
+Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
+
+## Read the results
+
+Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
+
+Use the batch selector or Scans tab to reopen previous results. Each batch keeps its own findings, settings, selected runs, coverage and cost. Older batches created before snapshot support remain available through accumulated findings. The Runs tab lists the selected batch's sample and can filter per-run observations, including runs without an observed issue and runs with insufficient evidence. These observations precede the final evidence investigation. Linked-run counts on findings include cited counterexamples, so they are not failure counts
+
+Choose **This is expected** and explain why to teach later scans about acceptable behavior. Feedback is kept with the lens and included in subsequent reviews. It does not alter historical evidence or exempt different problems
+
+## What a scan does
+
+The proxy selects executions received or updated within the configured lookback window, with a two-minute settling period. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID
+
+A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
+
+The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence
+
+There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools
+
+Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
+
+Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
+
+## Operations and limits
+
+PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
+
+Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Both the Lens budget and the selected virtual key’s budgets, model permissions, and rate limits apply. Analysis spend appears under that key in Virtual Keys and normal request logs, with Lens, scan, and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Existing workers need a billing key assigned in **Set up analysis** before they can resume
+
+V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them
+
+
+## API access
+
+The UI and API use the same scan lifecycle. Authenticate with a proxy administrator credential for writes, or a proxy-admin viewer credential for reads. Worker credentials are only for worker operations
+
+```bash
+curl "$LITELLM_URL/lens" -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H 'Content-Type: application/json' -d '{
+    "name": "Research quality", "model": "your-model-alias",
+    "context": "Answer the requested question using cited, retrieved evidence.",
+    "source": "traces", "lookback_hours": 24,
+    "sample_percent": 100, "sample_size": null, "concurrency": 8,
+    "enabled": true, "interval_minutes": 1440, "monthly_budget": 50
+  }'
+
+curl "$LITELLM_URL/lens/$LENS_ID/runs" -X POST \
+  -H "Authorization: Bearer $LITELLM_API_KEY" -H 'Content-Type: application/json' -d '{}'
+
+curl "$LITELLM_URL/lens/$LENS_ID/runs?offset=0" -H "Authorization: Bearer $LITELLM_API_KEY"
+curl "$LITELLM_URL/lens/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITELLM_API_KEY"
+```
+
+Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/lens/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /lens/{id}/findings/{finding_id}` with `status` and `reason`
+
+## Quality evaluation
+
+Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
+
+```bash
+python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
+  --model your-model-alias --split all --background 1000 --concurrency 16 \
+  --output /tmp/lens-quality.json
+```
+
+Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
+
+The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
+
+To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
+
+## Upgrading from the original Lens API
+
+The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`, and worker claims use `lens_id`. Upgrade the proxy and recreate every worker with the image shown by the upgraded dashboard before starting new scans. Update API clients to the new paths and response fields. Old worker images cannot poll the renamed API
+
+Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
+
+Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push

deploy/lens/compose.build.yaml

@@ -0,0 +1,6 @@
+services:
+  lens-worker:
+    build:
+      context: ../..
+      dockerfile: deploy/lens/Dockerfile
+    image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  lens-worker:
+    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:a8e8731d954916594eea462969946b9292fb771681ff515a9fd296b53f856c77}
+    environment:
+      LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
+      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
+    restart: unless-stopped
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]

docker/Dockerfile.non_root

@@ -103,7 +103,7 @@ ENV LITELLM_NON_ROOT=true
 
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
     cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
-    cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
+    cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \
     touch /var/lib/litellm/ui/.litellm_ui_ready
 
 RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \

docker/docker-compose.tracing.yml

@@ -0,0 +1,62 @@
+name: litellm-tracing
+
+services:
+  litellm:
+    build:
+      context: ..
+      target: runtime
+    command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
+    environment:
+      LITELLM_MASTER_KEY: local-tracing-master-key
+      LITELLM_SALT_KEY: sk-local-tracing-salt-key
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+      CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_READER_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_DATABASE: litellm
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+    volumes:
+      - ./tracing-config.yaml:/app/tracing-config.yaml:ro
+    ports:
+      - "127.0.0.1:4002:4000"
+    depends_on:
+      db:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:15432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:26.9.6.6
+    environment:
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: local-tracing
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    ports:
+      - "127.0.0.1:18123:8123"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+volumes:
+  postgres_data:
+  clickhouse_data:

docker/tracing-config.yaml

@@ -0,0 +1,10 @@
+model_list:
+  - model_name: gpt-6.1-sol
+    litellm_params:
+      model: openai/gpt-6.1-sol
+      api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  tracing:
+    store: clickhouse

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -32,6 +32,7 @@ from litellm.integrations.email_templates.key_rotated_email import (
 from litellm.integrations.email_templates.templates import (
     MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
     SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
+    TEAM_MEMBER_MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
     TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
 )
 from litellm.integrations.email_templates.user_invitation_email import (
@@ -48,6 +49,12 @@ from litellm.secret_managers.main import get_secret_bool
 from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
 
 
+def _max_budget_alert_id(user_info: CallInfo) -> str:
+    if user_info.event_group == Litellm_EntityType.TEAM_MEMBER:
+        return f"team_member:{user_info.user_id}:{user_info.team_id}"
+    return user_info.token or user_info.user_id or "default_id"
+
+
 def _parse_email_list(raw) -> List[str]:
     """Parse emails from a list or comma-separated string."""
     if isinstance(raw, list):
@@ -373,17 +380,31 @@ class BaseEmailLogger(CustomLogger):
             greeting = html.escape(
                 event.user_email or event.key_alias or event.token or ""
             )
-            email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
+            if event.event_group == Litellm_EntityType.TEAM_MEMBER:
-                email_logo_url=email_params.logo_url,
+                email_html_content = TEAM_MEMBER_MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
-                recipient_email=greeting,
+                    email_logo_url=email_params.logo_url,
-                percentage=percentage,
+                    member=html.escape(event.user_email or event.user_id or ""),
-                spend=spend_str,
+                    team_alias=html.escape(event.team_alias or event.team_id or ""),
-                max_budget=max_budget_str,
+                    percentage=percentage,
-                alert_threshold=alert_threshold_str,
+                    spend=spend_str,
-                base_url=email_params.base_url,
+                    max_budget=max_budget_str,
-                email_support_contact=email_params.support_contact,
+                    alert_threshold=alert_threshold_str,
-                email_footer=email_params.signature,
+                    base_url=email_params.base_url,
-            )
+                    email_support_contact=email_params.support_contact,
+                    email_footer=email_params.signature,
+                )
+            else:
+                email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
+                    email_logo_url=email_params.logo_url,
+                    recipient_email=greeting,
+                    percentage=percentage,
+                    spend=spend_str,
+                    max_budget=max_budget_str,
+                    alert_threshold=alert_threshold_str,
+                    base_url=email_params.base_url,
+                    email_support_contact=email_params.support_contact,
+                    email_footer=email_params.signature,
+                )
             await self.send_email(
                 from_email=self.DEFAULT_LITELLM_EMAIL,
                 to_email=recipient_emails,
@@ -607,7 +628,7 @@ class BaseEmailLogger(CustomLogger):
             if user_info.spend < threshold_amount:
                 continue
 
-            _id = user_info.token or user_info.user_id or "default_id"
+            _id = _max_budget_alert_id(user_info)
             _cache_key = (
                 f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
             )
@@ -618,7 +639,7 @@ class BaseEmailLogger(CustomLogger):
                 emails.append(user_info.user_email)
             if not emails:
                 verbose_proxy_logger.warning(
-                    "No recipients for %d%% threshold on key %s, skipping alert",
+                    "No recipients for %d%% threshold on %s, skipping alert",
                     threshold_pct,
                     _id,
                 )
@@ -633,7 +654,11 @@ class BaseEmailLogger(CustomLogger):
             if send_count is not None and send_count > 1:
                 continue
 
-            event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
+            event_message = (
+                f"Team Member Budget Alert - {threshold_pct}% of Team Member Budget Reached"
+                if user_info.event_group == Litellm_EntityType.TEAM_MEMBER
+                else f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
+            )
             webhook_event = WebhookEvent(
                 event="max_budget_alert",
                 event_message=event_message,

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,10 +22,8 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management_endpoints.common_utils import (
+from litellm.proxy.management.teams.access import is_team_admin
-    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # shared owner of team-admin membership
+from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
-    _set_object_metadata_field,
-)
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
@@ -117,7 +115,7 @@ async def _check_user_permission_for_project(
         return False
 
     team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
-    return _is_user_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
+    return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
 
 
 async def _validate_team_exists(

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.71"
+version = "0.1.72"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.71"
+version = "0.1.72"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -73,6 +73,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/containers",
     "/containers",
     "/v1/evals",
+    "/v1/traces",
     "/v1/memory",
     "/queue/chat/",
     # Google data plane (v1beta is the Google AI Studio version)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260921190000_agent_identity/migration.sql

@@ -0,0 +1,97 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "enabled" BOOLEAN NOT NULL DEFAULT true,
+ADD COLUMN IF NOT EXISTS "execution_mode" TEXT NOT NULL DEFAULT 'autonomous',
+ADD COLUMN IF NOT EXISTS "identity_managed" BOOLEAN NOT NULL DEFAULT false;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "billing_agent_id" TEXT;
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AgentIdentity" (
+    "agent_id" TEXT NOT NULL,
+    "active" BOOLEAN NOT NULL DEFAULT true,
+    "provider" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "client_id" TEXT NOT NULL,
+    "service_principal_id" TEXT,
+    "required_roles" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "required_scopes" TEXT[] DEFAULT ARRAY['user_impersonation']::TEXT[],
+    "revision" TEXT NOT NULL,
+    "last_authenticated_at" TIMESTAMP(3),
+
+    CONSTRAINT "LiteLLM_AgentIdentity_pkey" PRIMARY KEY ("agent_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgentIdentity" (
+    "binding_id" TEXT NOT NULL,
+    "agent_id" TEXT,
+    "provider" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "client_id" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_RetiredAgentIdentity_pkey" PRIMARY KEY ("binding_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgent" (
+    "original_agent_id" TEXT NOT NULL,
+    "retired_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_RetiredAgent_pkey" PRIMARY KEY ("original_agent_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_VerifiedSubject" (
+    "subject_id" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "oid" TEXT NOT NULL,
+    "kind" TEXT NOT NULL DEFAULT 'human',
+    "user_id" TEXT,
+    "verified_via" TEXT NOT NULL DEFAULT 'sso_interactive',
+    "verified_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_VerifiedSubject_pkey" PRIMARY KEY ("subject_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_AgentIdentity"("provider", "tenant_id", "client_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_issuer_service_principal_id_key" ON "LiteLLM_AgentIdentity"("issuer", "service_principal_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_RetiredAgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_RetiredAgentIdentity"("provider", "tenant_id", "client_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_user_id_idx" ON "LiteLLM_VerifiedSubject"("user_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_issuer_tenant_id_oid_key" ON "LiteLLM_VerifiedSubject"("issuer", "tenant_id", "oid");
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_AgentIdentity_agent_id_fkey') THEN
+        ALTER TABLE "LiteLLM_AgentIdentity" ADD CONSTRAINT "LiteLLM_AgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE CASCADE ON UPDATE CASCADE;
+    END IF;
+END $$;
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_RetiredAgentIdentity_agent_id_fkey') THEN
+        ALTER TABLE "LiteLLM_RetiredAgentIdentity" ADD CONSTRAINT "LiteLLM_RetiredAgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_VerifiedSubject_user_id_fkey') THEN
+        ALTER TABLE "LiteLLM_VerifiedSubject" ADD CONSTRAINT "LiteLLM_VerifiedSubject_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE CASCADE ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260925000000_add_mcp_pinned_tools/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "pinned_tools" JSONB DEFAULT '{}';

litellm-proxy-extras/litellm_proxy_extras/migrations/20260928000000_add_daily_model_usage/migration.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyModelUsage" (
+    "date" TEXT NOT NULL,
+    "model_group" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "task_type" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "prompt_tokens" BIGINT NOT NULL DEFAULT 0,
+    "completion_tokens" BIGINT NOT NULL DEFAULT 0,
+    "request_count" BIGINT NOT NULL DEFAULT 0,
+    "successful_requests" BIGINT NOT NULL DEFAULT 0,
+    "failed_requests" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    CONSTRAINT "LiteLLM_DailyModelUsage_pkey" PRIMARY KEY ("date", "model_group", "model", "custom_llm_provider", "task_type")
+);
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyModelUsage_date_idx" ON "LiteLLM_DailyModelUsage"("date");
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyModelUsage_model_group_idx" ON "LiteLLM_DailyModelUsage"("model_group");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_Engine" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "version" INTEGER NOT NULL DEFAULT 0,
+    "data" JSONB NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "LiteLLM_EngineWorker" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "token_hash" TEXT NOT NULL UNIQUE,
+    "data" JSONB NOT NULL
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261001000000_lens_run_history/migration.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_EngineRun" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "engine_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL,
+    "data" JSONB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_EngineRun_engine_id_created_at_idx" ON "LiteLLM_EngineRun"("engine_id", "created_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql

@@ -0,0 +1,18 @@
+DO $$
+BEGIN
+    ALTER TABLE IF EXISTS "LiteLLM_Engine" RENAME TO "LiteLLM_Lens";
+    ALTER TABLE IF EXISTS "LiteLLM_EngineRun" RENAME TO "LiteLLM_LensRun";
+    ALTER TABLE IF EXISTS "LiteLLM_EngineWorker" RENAME TO "LiteLLM_LensWorker";
+    IF EXISTS (
+        SELECT 1 FROM pg_attribute
+        WHERE attrelid = to_regclass('"LiteLLM_LensRun"')
+          AND attname = 'engine_id' AND NOT attisdropped
+    ) THEN
+        ALTER TABLE "LiteLLM_LensRun" RENAME COLUMN "engine_id" TO "lens_id";
+    END IF;
+    ALTER INDEX IF EXISTS "LiteLLM_Engine_pkey" RENAME TO "LiteLLM_Lens_pkey";
+    ALTER INDEX IF EXISTS "LiteLLM_EngineRun_pkey" RENAME TO "LiteLLM_LensRun_pkey";
+    ALTER INDEX IF EXISTS "LiteLLM_EngineWorker_pkey" RENAME TO "LiteLLM_LensWorker_pkey";
+    ALTER INDEX IF EXISTS "LiteLLM_EngineWorker_token_hash_key" RENAME TO "LiteLLM_LensWorker_token_hash_key";
+    ALTER INDEX IF EXISTS "LiteLLM_EngineRun_engine_id_created_at_idx" RENAME TO "LiteLLM_LensRun_lens_id_created_at_idx";
+END $$;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -78,6 +78,11 @@ model LiteLLM_AgentsTable {
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
+  identity_managed Boolean @default(false)
+  enabled Boolean @default(true)
+  execution_mode String @default("autonomous")
+  identity LiteLLM_AgentIdentity?
+  retired_identities LiteLLM_RetiredAgentIdentity[]
   tpm_limit  Int?
   rpm_limit  Int?
   session_tpm_limit Int?
@@ -88,6 +93,56 @@ model LiteLLM_AgentsTable {
   updated_by String
 }
 
+model LiteLLM_AgentIdentity {
+  agent_id String @id
+  active Boolean @default(true)
+  agent LiteLLM_AgentsTable @relation(fields: [agent_id], references: [agent_id], onDelete: Cascade)
+  provider String
+  issuer String
+  tenant_id String
+  client_id String
+  service_principal_id String?
+  required_roles String[] @default([])
+  required_scopes String[] @default(["user_impersonation"])
+  revision String @default(uuid())
+  last_authenticated_at DateTime?
+  @@unique([provider, tenant_id, client_id])
+  @@unique([issuer, service_principal_id])
+}
+
+model LiteLLM_RetiredAgentIdentity {
+  binding_id String @id @default(uuid())
+  agent_id String?
+  agent LiteLLM_AgentsTable? @relation(fields: [agent_id], references: [agent_id], onDelete: SetNull)
+  provider String
+  issuer String
+  tenant_id String
+  client_id String
+  @@unique([provider, tenant_id, client_id])
+}
+
+model LiteLLM_RetiredAgent {
+  original_agent_id String @id
+  retired_at DateTime @default(now())
+}
+
+model LiteLLM_VerifiedSubject {
+  subject_id String @id @default(uuid())
+  issuer String
+  tenant_id String
+  oid String
+  kind String @default("human")
+  user_id String?
+  user LiteLLM_UserTable? @relation(fields: [user_id], references: [user_id], onDelete: Cascade)
+  verified_via String @default("sso_interactive")
+  verified_at DateTime @default(now())
+  @@unique([issuer, tenant_id, oid])
+  @@index([user_id])
+}
+
+
+
+
 model LiteLLM_OrganizationTable {
 		organization_id String @id @default(uuid())
     organization_alias  String
@@ -241,6 +296,7 @@ model LiteLLM_DeletedTeamTable {
 
 // Track spend, rate limit, budget Users
 model LiteLLM_UserTable {
+    verified_subjects LiteLLM_VerifiedSubject[]
 		user_id    String @id
     user_alias String? 
     team_id    String?
@@ -322,6 +378,7 @@ model LiteLLM_MCPServerTable {
   allowed_tools String[]      @default([])
   tool_name_to_display_name Json? @default("{}")
   tool_name_to_description   Json? @default("{}")
+  pinned_tools Json? @default("{}")
   extra_headers String[]    @default([])
   static_headers Json?        @default("{}")
   // Admin-configured environment variables interpolated into static_headers
@@ -674,6 +731,7 @@ model LiteLLM_SpendLogs {
   session_id          String?
   status              String?
   mcp_namespaced_tool_name String? 
+  billing_agent_id    String?
   agent_id            String?
   proxy_server_request Json?     @default("{}")
   litellm_call_id     String?
@@ -1259,6 +1317,26 @@ model LiteLLM_DailyToolSpend {
   @@id([date, tool_name])
 }
 
+model LiteLLM_DailyModelUsage {
+  date                String
+  model_group         String
+  model               String
+  custom_llm_provider String
+  task_type           String
+  spend               Float    @default(0.0)
+  prompt_tokens       BigInt   @default(0)
+  completion_tokens   BigInt   @default(0)
+  request_count       BigInt   @default(0)
+  successful_requests BigInt   @default(0)
+  failed_requests     BigInt   @default(0)
+  created_at          DateTime @default(now())
+  updated_at          DateTime @updatedAt
+
+  @@id([date, model_group, model, custom_llm_provider, task_type])
+  @@index([date])
+  @@index([model_group])
+}
+
 // Gateway request counts recorded at the ASGI edge by
 // BillableRequestMetricsMiddleware. This is the source of truth for SGR
 // (successful gateway requests): it counts what the proxy actually answered,
@@ -1816,3 +1894,24 @@ model LiteLLM_WorkflowMessage {
   @@unique([run_id, sequence_number])
   @@index([run_id])
 }
+
+model LiteLLM_Lens {
+  id      String @id
+  version Int @default(0)
+  data    Json
+}
+
+model LiteLLM_LensRun {
+  id         String @id
+  lens_id  String
+  created_at DateTime
+  data       Json
+
+  @@index([lens_id, created_at])
+}
+
+model LiteLLM_LensWorker {
+  id         String @id
+  token_hash String @unique
+  data       Json
+}

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -590,6 +590,36 @@ class ProxyExtrasDBManager:
                         f"Failed to resolve migration {migration_name}: {e.stderr}"
                     )
 
+    @staticmethod
+    def raise_if_lens_rename_pending() -> None:
+        database_url: Final = os.environ.get("DATABASE_URL")
+        if not database_url:
+            return
+        try:
+            import psycopg
+        except ImportError as exc:
+            raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc
+        try:
+            with psycopg.connect(
+                ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True
+            ) as connection:
+                legacy: Final = connection.execute(
+                    "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace "
+                    "WHERE n.nspname=%s AND c.relname IN ('LiteLLM_Engine', 'LiteLLM_EngineRun', 'LiteLLM_EngineWorker') "
+                    "LIMIT 1",
+                    (ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",),
+                ).fetchone()
+        except psycopg.Error as exc:
+            raise RuntimeError(
+                "Cannot verify Lens data safety; refusing prisma db push. Check database connectivity and psycopg installation."
+            ) from exc
+        if legacy is not None:
+            raise RuntimeError(
+                "Legacy Lens tables exist. prisma db push would drop saved Lens data. "
+                "Apply the shipped 20261001100000_rename_lens migration to this database schema before retrying. "
+                "Deployments using migration history can upgrade without --use_prisma_db_push instead."
+            )
+
     @staticmethod
     def spend_logs_is_partitioned() -> bool:
         """True when the connected database's LiteLLM_SpendLogs is a
@@ -895,6 +925,7 @@ class ProxyExtrasDBManager:
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
 
         if not use_migrate:
+            ProxyExtrasDBManager.raise_if_lens_rename_pending()
             if ProxyExtrasDBManager.spend_logs_is_partitioned():
                 raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
             original_dir = os.getcwd()
@@ -1398,6 +1429,7 @@ class ProxyExtrasDBManager:
                     if ProxyExtrasDBManager.spend_logs_is_partitioned():
                         raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
+                    ProxyExtrasDBManager.raise_if_lens_rename_pending()
                     prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                         timeout=prisma_command_timeout(),

litellm-proxy-extras/pyproject.toml

@@ -1,9 +1,13 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.102"
+version = "0.4.103"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
+dependencies = [
+    "psycopg>=3.2,<4.0",
+    "psycopg-binary>=3.2,<4.0",
+]
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [
@@ -26,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.102"
+version = "0.4.103"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/.agents/skills/rust-tracing/SKILL.md

@@ -0,0 +1,22 @@
+---
+name: rust-tracing
+description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
+---
+
+# Rust tracing
+
+Use upstream `tracing` throughout Rust, including `#[tracing::instrument]`, events, and span propagation. Centralize collection and delivery infrastructure in `crates/tracing`. Direct upstream imports still reach our configured subscriber; re-exporting macros does not control delivery. Do not introduce Rust `log` or `pyo3-log` for this path
+
+`litellm-tracing` owns shared subscriber layers, span field collection, and diagnostic processing. Keep adapters composable as `tracing_subscriber::Layer`s, with `Logger` providing host setup. Runtime-specific delivery belongs in the host bridge. The Python bridge delivers directly to the existing Python SDK logger, preserving its handlers, filtering, redaction, and request correlation. Keep Python dependencies out of `crates/tracing`
+
+Hosts configure subscribers. Keep Python execution scoped to its captured dispatch rather than installing a process-wide subscriber. Propagate both span context and dispatch across spawned work and returned streams
+
+In core, instrument execution shared by native calls and hosted machines. Use consistent route, model, provider, streaming, and outcome fields. Put status recording at shared provider boundaries instead of scattering basic logging through handlers. Keep upstream HTTP status separate from route success
+
+Use `skip_all` and explicitly selected fields. Basic tracing excludes bodies, credentials, headers, and raw error strings. Avoid automatic `ret` or `err` capture of sensitive values. Keep payload diagnostics separate and subject to existing redaction
+
+A returned stream retains its route span until exhaustion, error, or drop, with exactly one terminal outcome. Builder construction does not start a trace. Never hold a span entry guard across an await. Diagnostic tracing remains separate from lifecycle callbacks and `CustomLogger` dispatch
+
+Use `litellm_tracing::sink_layer` to compose a sink with other subscriber layers. It inherits span fields into events and emits span-close summaries with elapsed time. Test observable records, concurrent isolation, dynamic filtering, sensitive-field exclusion, and stream cancellation when changing this behavior
+
+Consult the [tracing API](https://docs.rs/tracing/latest/tracing/) and [subscriber layers](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/layer/index.html) for implementation details

litellm-rust/AGENTS.md

@@ -1,5 +1,7 @@
 # Rust workspace rules
 
+For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.agents/skills/rust-tracing/SKILL.md)
+
 ## Test placement
 
 - Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
@@ -9,10 +11,16 @@
 - A test for another crate's item belongs in that crate, not in a downstream one
 - Never set `autotests = false` or hand-list `[[test]]` targets; every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own
 
+## Test fixtures and cases
+
+Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
+
 ## Error definitions
 
 - A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`
-- Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each
+- Put message templates in the variant's `#[error(...)]` declaration. Callers pass only the small typed arguments needed to fill them, never `Error::Variant(format!(...))` or a preformatted message. Keep the smallest set of neutral variants that callers need to distinguish; different wording or providers do not justify new variants
+- Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each. A failure mode is something a caller handles differently (phase, status code, retry, a message Python parity pins exactly); failures no caller tells apart share one variant and differ only in its message
+- Keep shared error enums minimal and provider-neutral. Provider names, credential types, configuration fields, and setup guidance belong in caller-supplied data, not dedicated variants or hardcoded shared messages. Reuse a variant for the same failure mode across providers, such as `MissingApiBase { provider: "Azure", guidance: "..." }`. An exact parity message does not justify a provider-specific variant when caller-supplied context can preserve it
 - Wrap a lower-level error as a variant with `#[from]` or `#[source]` instead of flattening it to a string
 - Exception: split into separate types when different functions fail in disjoint ways, especially when different callers see them. A shared enum would force every caller to match variants its function can never return
 - Name a split type after what went wrong (a unit struct is fine for a single failure mode), not after the function that returns it

litellm-rust/Cargo.toml

@@ -9,11 +9,21 @@ license = "MIT"
 repository = "https://github.com/BerriAI/litellm"
 
 [workspace.dependencies]
+litellm-config = { path = "crates/config" }
+litellm-router = { path = "crates/router" }
 litellm-tracing = { path = "crates/tracing" }
-tracing = "0.1"
+litellm-traces = { path = "crates/traces" }
 litellm-core = { path = "crates/core" }
+litellm-gateway-mcp = { path = "crates/gateway-mcp" }
+litellm-gateway = { path = "crates/gateway" }
+litellm-gateway-inference = { path = "crates/gateway-inference" }
+litellm-gateway-auth = { path = "crates/gateway-auth" }
+litellm-gateway-management = { path = "crates/gateway-management" }
+litellm-gateway-ui = { path = "crates/gateway-ui" }
 litellm-coroutine = { path = "crates/coroutine" }
 litellm-host = { path = "crates/host" }
+litellm-host-http = { path = "crates/host-http" }
+litellm-host-native = { path = "crates/host-native" }
 litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
 litellm-framing = { path = "crates/framer" }
 litellm-auth = { path = "crates/auth" }
@@ -30,8 +40,10 @@ litellm-secrets-azure = { path = "crates/secrets-azure" }
 litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
-litellm-types = { path = "crates/types" }
+litellm-llms-types = { path = "crates/llms-types" }
 litellm-core-utils = { path = "crates/core-utils" }
+litellm-db = { path = "crates/db" }
+litellm-db-testing = { path = "crates/db-testing" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
 litellm-cache-memory = { path = "crates/cache-memory" }
@@ -48,7 +60,12 @@ litellm-token-counter-fast = { path = "crates/token-counter-fast" }
 litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
 litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
 litellm-host-python = { path = "crates/host-python" }
+litellm-python-compat = { path = "crates/python-compat" }
 
+tracing = "0.1"
+axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }
+axum-login = "0.18.0"
+tower-sessions = { version = "0.14.0", features = ["memory-store"] }
 bytes = "1"
 http = "1"
 google-cloud-auth = { version = "1.16.0", default-features = false }
@@ -57,8 +74,9 @@ hyper-util = { version = "0.1.20", default-features = false, features = ["client
 proptest = "1.7.0"
 pyo3 = "0.29.2"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
-pythonize = "0.29.0"
 rand = "0.8"
+macro_rules_attribute = "0.2.3"
+schemars = "1"
 reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] }
 qdrant-client = { version = "1.19.0", default-features = false }
 uuid = { version = "1", features = ["v4"] }
@@ -73,6 +91,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = { version = "=3.16.1", default-features = false, features = ["std", "macros"] }
 sha2 = "0.10"
+sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
 subtle = "2"
 thiserror = "2.0"
 tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }

litellm-rust/clippy.toml

@@ -1,4 +1,4 @@
-# The Tokio runtime is reached only through `host-python/src/execution.rs`, whose fork gate
+# The Tokio runtime is reached only through `host-python/src/runtime.rs`, whose fork gate
 # must see every entry. Going around it makes a fork-after-use hang instead of raising.
 disallowed-methods = [
     { path = "pyo3_async_runtimes::tokio::get_runtime", reason = "use litellm_host_python::run_sync / run_sync_value" },
@@ -7,4 +7,23 @@ disallowed-methods = [
     { path = "pyo3_async_runtimes::tokio::local_future_into_py", reason = "use litellm_host_python::run_async / run_async_value" },
     { path = "pyo3_async_runtimes::tokio::run", reason = "use litellm_host_python::run_sync / run_sync_value" },
     { path = "pyo3_async_runtimes::tokio::run_until_complete", reason = "use litellm_host_python::run_sync / run_sync_value" },
+    { path = "reqwest::Client::new", reason = "take litellm_http::Client from HttpClientPool" },
+    { path = "reqwest::Client::builder", reason = "HttpClientConfig owns client construction" },
+    { path = "reqwest::ClientBuilder::danger_accept_invalid_certs", reason = "set HttpClientConfig::verify instead" },
+    { path = "reqwest::ClientBuilder::identity", reason = "set HttpClientConfig::client_certificate instead" },
+    { path = "reqwest::ClientBuilder::use_preconfigured_tls", reason = "HttpClientConfig owns the TLS configuration" },
+    { path = "sqlx::query", reason = "use sqlx::query! or query_file! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::query_as", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::query_scalar", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::query_with", reason = "use sqlx::query! or query_file! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::query_as_with", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::query_scalar_with", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" },
+    { path = "sqlx::raw_sql", reason = "raw_sql is unchecked; use the checked query macros" },
+]
+
+# Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS,
+# proxy and timeout settings. Only crates/http builds one.
+disallowed-types = [
+    { path = "reqwest::Client", reason = "take litellm_http::Client from HttpClientPool; only crates/http builds one" },
+    { path = "reqwest::ClientBuilder", reason = "HttpClientConfig owns client construction" },
 ]

litellm-rust/crates/auth-aws/Cargo.toml

@@ -22,5 +22,7 @@ aws-types = "1.4.0"
 aws-smithy-runtime-api = "1.13.0"
 
 [dev-dependencies]
+rstest.workspace = true
+litellm-http = { workspace = true, features = ["test-support"] }
 reqwest.workspace = true
 tokio.workspace = true

litellm-rust/crates/auth-aws/src/aws.rs

@@ -1,5 +1,4 @@
 use std::collections::BTreeMap;
-use std::sync::OnceLock;
 use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -26,8 +25,26 @@ use super::constants::{
 const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
 const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600);
 
-static STATIC_CREDENTIALS_CACHE: OnceLock<Cache<String, Credentials>> = OnceLock::new();
+#[derive(Clone)]
-static AMBIENT_CREDENTIALS_CACHE: OnceLock<Cache<String, Credentials>> = OnceLock::new();
+pub struct AwsAuthService {
+    static_credentials: Cache<String, Credentials>,
+    ambient_credentials: Cache<String, Credentials>,
+}
+
+impl Default for AwsAuthService {
+    fn default() -> Self {
+        Self {
+            static_credentials: Cache::builder()
+                .max_capacity(200)
+                .time_to_live(STATIC_CREDENTIALS_TTL)
+                .build(),
+            ambient_credentials: Cache::builder()
+                .max_capacity(200)
+                .time_to_live(AMBIENT_CREDENTIALS_TTL)
+                .build(),
+        }
+    }
+}
 
 fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option<Duration> {
     match flow {
@@ -108,35 +125,19 @@ fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String {
     format!("{:x}", hasher.finalize())
 }
 
-fn static_credentials_cache() -> &'static Cache<String, Credentials> {
+impl AwsAuthService {
-    STATIC_CREDENTIALS_CACHE.get_or_init(|| {
+    fn get_cached_credentials(&self, key: &str) -> Option<Credentials> {
-        Cache::builder()
+        self.static_credentials
-            .max_capacity(200)
+            .get(key)
-            .time_to_live(STATIC_CREDENTIALS_TTL)
+            .or_else(|| self.ambient_credentials.get(key))
-            .build()
+    }
-    })
-}
 
-fn ambient_credentials_cache() -> &'static Cache<String, Credentials> {
+    fn set_cached_credentials(&self, key: String, credentials: Credentials, ttl: Duration) {
-    AMBIENT_CREDENTIALS_CACHE.get_or_init(|| {
+        if ttl == STATIC_CREDENTIALS_TTL {
-        Cache::builder()
+            self.static_credentials.insert(key, credentials);
-            .max_capacity(200)
+        } else {
-            .time_to_live(AMBIENT_CREDENTIALS_TTL)
+            self.ambient_credentials.insert(key, credentials);
-            .build()
+        }
-    })
-}
-
-fn get_cached_credentials(key: &str) -> Option<Credentials> {
-    static_credentials_cache()
-        .get(key)
-        .or_else(|| ambient_credentials_cache().get(key))
-}
-
-fn set_cached_credentials(key: String, credentials: Credentials, ttl: Duration) {
-    if ttl == STATIC_CREDENTIALS_TTL {
-        static_credentials_cache().insert(key, credentials);
-    } else {
-        ambient_credentials_cache().insert(key, credentials);
     }
 }
 
@@ -214,66 +215,157 @@ pub fn classify_auth(
     AwsAuthFlow::DefaultChain
 }
 
-pub async fn resolve_credentials(
+impl AwsAuthService {
-    config: AwsAuthConfig,
+    pub async fn resolve_credentials(
-    env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+        &self,
-) -> Result<Credentials, Error> {
+        config: AwsAuthConfig,
-    let resolved = config.clone().with_environment(env_lookup);
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    let flow = classify_auth(config, env_lookup);
+    ) -> Result<Credentials, Error> {
-    match flow {
+        let resolved = config.clone().with_environment(env_lookup);
-        AwsAuthFlow::SessionToken {
+        let flow = classify_auth(config, env_lookup);
-            access_key_id,
+        match flow {
-            secret_access_key,
+            AwsAuthFlow::SessionToken {
-            session_token,
-        } => Ok(Credentials::new(
-            access_key_id,
-            secret_access_key,
-            Some(session_token),
-            None,
-            "litellm-static-session",
-        )),
-        AwsAuthFlow::StaticKeys {
-            access_key_id,
-            secret_access_key,
-            region_name,
-        } => {
-            let flow = AwsAuthFlow::StaticKeys {
-                access_key_id: access_key_id.clone(),
-                secret_access_key: secret_access_key.clone(),
-                region_name,
-            };
-            let key = cache_key(&resolved, &flow);
-            if let Some(credentials) = get_cached_credentials(&key) {
-                return Ok(credentials);
-            }
-            let credentials = Credentials::new(
                 access_key_id,
                 secret_access_key,
+                session_token,
+            } => Ok(Credentials::new(
+                access_key_id,
+                secret_access_key,
+                Some(session_token),
                 None,
-                None,
+                "litellm-static-session",
-                "litellm-static",
+            )),
-            );
+            AwsAuthFlow::StaticKeys {
-            set_cached_credentials(
+                access_key_id,
-                key,
+                secret_access_key,
-                credentials.clone(),
+                region_name,
-                credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL),
+            } => {
-            );
+                let flow = AwsAuthFlow::StaticKeys {
-            Ok(credentials)
+                    access_key_id: access_key_id.clone(),
-        }
+                    secret_access_key: secret_access_key.clone(),
-        AwsAuthFlow::Profile { name } => {
+                    region_name,
-            let provider = aws_config::profile::ProfileFileCredentialsProvider::builder()
+                };
-                .profile_name(name)
+                let key = cache_key(&resolved, &flow);
-                .build();
+                if let Some(credentials) = self.get_cached_credentials(&key) {
-            provider
+                    return Ok(credentials);
-                .provide_credentials()
+                }
-                .await
+                let credentials = Credentials::new(
-                .map_err(|error| Error::AwsProfile(error.to_string()))
+                    access_key_id,
-        }
+                    secret_access_key,
-        AwsAuthFlow::AssumeRole { role, session_name } => {
+                    None,
-            if is_already_running_as_role(&role, &resolved).await? {
+                    None,
-                let ambient_flow = AwsAuthFlow::DefaultChain;
+                    "litellm-static",
-                let key = cache_key(&resolved, &ambient_flow);
+                );
-                if let Some(credentials) = get_cached_credentials(&key) {
+                self.set_cached_credentials(
+                    key,
+                    credentials.clone(),
+                    credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL),
+                );
+                Ok(credentials)
+            }
+            AwsAuthFlow::Profile { name } => {
+                let provider = aws_config::profile::ProfileFileCredentialsProvider::builder()
+                    .profile_name(name)
+                    .build();
+                provider
+                    .provide_credentials()
+                    .await
+                    .map_err(|error| Error::AwsProfile(error.to_string()))
+            }
+            AwsAuthFlow::AssumeRole { role, session_name } => {
+                if is_already_running_as_role(&role, &resolved).await? {
+                    let ambient_flow = AwsAuthFlow::DefaultChain;
+                    let key = cache_key(&resolved, &ambient_flow);
+                    if let Some(credentials) = self.get_cached_credentials(&key) {
+                        return Ok(credentials);
+                    }
+                    let provider =
+                    aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
+                        .build()
+                        .await;
+                    let credentials = provider
+                        .provide_credentials()
+                        .await
+                        .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
+                    self.set_cached_credentials(
+                        key,
+                        credentials.clone(),
+                        credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
+                    );
+                    return Ok(credentials);
+                }
+                let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
+                if let Some(region) = resolved.region_name.clone() {
+                    loader = loader.region(aws_types::region::Region::new(region));
+                }
+                if let Some(endpoint) = resolved.sts_endpoint.clone() {
+                    loader = loader.endpoint_url(endpoint);
+                }
+                if let (Some(access_key_id), Some(secret_access_key)) =
+                    (resolved.access_key_id, resolved.secret_access_key)
+                {
+                    loader = loader.credentials_provider(Credentials::new(
+                        access_key_id,
+                        secret_access_key,
+                        resolved.session_token,
+                        None,
+                        "litellm-role-source",
+                    ));
+                }
+                let sdk_config = loader.load().await;
+                let builder = aws_config::sts::AssumeRoleProvider::builder(role);
+                let builder = match session_name {
+                    Some(name) => builder.session_name(name),
+                    None => builder.session_name(default_session_name()),
+                };
+                let builder = match resolved.external_id {
+                    Some(id) => builder.external_id(id),
+                    None => builder,
+                };
+                let provider = builder.configure(&sdk_config).build().await;
+                provider
+                    .provide_credentials()
+                    .await
+                    .map_err(|error| Error::AwsAssumeRole(error.to_string()))
+            }
+            AwsAuthFlow::WebIdentity {
+                token,
+                role,
+                session_name,
+            } => {
+                let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
+                if let Some(region) = resolved.region_name {
+                    loader = loader.region(aws_types::region::Region::new(region));
+                }
+                if let Some(endpoint) = resolved.sts_endpoint {
+                    loader = loader.endpoint_url(endpoint);
+                }
+                let sdk_config = loader.load().await;
+                let client = aws_sdk_sts::Client::new(&sdk_config);
+                let response = client
+                    .assume_role_with_web_identity()
+                    .role_arn(role)
+                    .role_session_name(session_name)
+                    .web_identity_token(token)
+                    .send()
+                    .await
+                    .map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
+                let credentials = response
+                    .credentials()
+                    .ok_or(Error::AwsMissingWebIdentityCredentials)?;
+                let expiration = SystemTime::try_from(*credentials.expiration())
+                    .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
+                Ok(Credentials::new(
+                    credentials.access_key_id(),
+                    credentials.secret_access_key(),
+                    Some(credentials.session_token().to_string()),
+                    Some(expiration),
+                    "litellm-web-identity",
+                ))
+            }
+            AwsAuthFlow::DefaultChain => {
+                let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
+                if let Some(credentials) = self.get_cached_credentials(&key) {
                     return Ok(credentials);
                 }
                 let provider =
@@ -284,101 +376,14 @@ pub async fn resolve_credentials(
                     .provide_credentials()
                     .await
                     .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
-                set_cached_credentials(
+                self.set_cached_credentials(
                     key,
                     credentials.clone(),
-                    credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
+                    credential_cache_ttl(&AwsAuthFlow::DefaultChain)
+                        .unwrap_or(AMBIENT_CREDENTIALS_TTL),
                 );
-                return Ok(credentials);
+                Ok(credentials)
             }
-            let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
-            if let Some(region) = resolved.region_name.clone() {
-                loader = loader.region(aws_types::region::Region::new(region));
-            }
-            if let Some(endpoint) = resolved.sts_endpoint.clone() {
-                loader = loader.endpoint_url(endpoint);
-            }
-            if let (Some(access_key_id), Some(secret_access_key)) =
-                (resolved.access_key_id, resolved.secret_access_key)
-            {
-                loader = loader.credentials_provider(Credentials::new(
-                    access_key_id,
-                    secret_access_key,
-                    resolved.session_token,
-                    None,
-                    "litellm-role-source",
-                ));
-            }
-            let sdk_config = loader.load().await;
-            let builder = aws_config::sts::AssumeRoleProvider::builder(role);
-            let builder = match session_name {
-                Some(name) => builder.session_name(name),
-                None => builder.session_name(default_session_name()),
-            };
-            let builder = match resolved.external_id {
-                Some(id) => builder.external_id(id),
-                None => builder,
-            };
-            let provider = builder.configure(&sdk_config).build().await;
-            provider
-                .provide_credentials()
-                .await
-                .map_err(|error| Error::AwsAssumeRole(error.to_string()))
-        }
-        AwsAuthFlow::WebIdentity {
-            token,
-            role,
-            session_name,
-        } => {
-            let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
-            if let Some(region) = resolved.region_name {
-                loader = loader.region(aws_types::region::Region::new(region));
-            }
-            if let Some(endpoint) = resolved.sts_endpoint {
-                loader = loader.endpoint_url(endpoint);
-            }
-            let sdk_config = loader.load().await;
-            let client = aws_sdk_sts::Client::new(&sdk_config);
-            let response = client
-                .assume_role_with_web_identity()
-                .role_arn(role)
-                .role_session_name(session_name)
-                .web_identity_token(token)
-                .send()
-                .await
-                .map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
-            let credentials = response
-                .credentials()
-                .ok_or(Error::AwsMissingWebIdentityCredentials)?;
-            let expiration = SystemTime::try_from(*credentials.expiration())
-                .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
-            Ok(Credentials::new(
-                credentials.access_key_id(),
-                credentials.secret_access_key(),
-                Some(credentials.session_token().to_string()),
-                Some(expiration),
-                "litellm-web-identity",
-            ))
-        }
-        AwsAuthFlow::DefaultChain => {
-            let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
-            if let Some(credentials) = get_cached_credentials(&key) {
-                return Ok(credentials);
-            }
-            let provider =
-                aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
-                    .build()
-                    .await;
-            let credentials = provider
-                .provide_credentials()
-                .await
-                .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
-            set_cached_credentials(
-                key,
-                credentials.clone(),
-                credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL),
-            );
-            Ok(credentials)
         }
     }
 }
@@ -585,6 +590,37 @@ pub fn aws_auth_config(
     }
 }
 
+/// Where the credentials that sign a request come from, decided when the request is
+/// prepared and resolved when it is sent.
+#[derive(Clone, Debug, PartialEq)]
+pub enum AwsCredentialSource {
+    HostSupplied(Credentials),
+    Chain(AwsAuthConfig),
+}
+
+impl AwsCredentialSource {
+    pub fn from_params(
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Self {
+        match host_supplied_credentials(optional_params) {
+            Some(credentials) => Self::HostSupplied(credentials),
+            None => Self::Chain(aws_auth_config(optional_params, env_lookup)),
+        }
+    }
+
+    pub async fn resolve(
+        self,
+        auth: &AwsAuthService,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Credentials, Error> {
+        match self {
+            Self::HostSupplied(credentials) => Ok(credentials),
+            Self::Chain(config) => auth.resolve_credentials(config, env_lookup).await,
+        }
+    }
+}
+
 /// Credentials a host resolved through its own chain and handed down verbatim.
 ///
 /// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads
@@ -747,17 +783,18 @@ mod tests {
 
     #[tokio::test]
     async fn static_credentials_do_not_use_network() {
-        let credentials = resolve_credentials(
+        let credentials = AwsAuthService::default()
-            AwsAuthConfig {
+            .resolve_credentials(
-                access_key_id: Some("ak".into()),
+                AwsAuthConfig {
-                secret_access_key: Some("sk".into()),
+                    access_key_id: Some("ak".into()),
-                region_name: Some("us-east-1".into()),
+                    secret_access_key: Some("sk".into()),
-                ..Default::default()
+                    region_name: Some("us-east-1".into()),
-            },
+                    ..Default::default()
-            &no_env,
+                },
-        )
+                &no_env,
-        .await
+            )
-        .expect("static credentials");
+            .await
+            .expect("static credentials");
         assert_eq!(credentials.access_key_id(), "ak");
         assert_eq!(credentials.session_token(), None);
     }
@@ -807,17 +844,67 @@ mod tests {
         );
     }
 
-    #[test]
+    #[rstest::rstest]
     fn cache_round_trip_preserves_credentials() {
+        let auth = AwsAuthService::default();
         let key = format!("cache-test-{}", std::process::id());
         let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test");
-        set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL);
+        auth.set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL);
         assert_eq!(
-            get_cached_credentials(&key).map(|value| value.access_key_id().to_string()),
+            auth.get_cached_credentials(&key)
+                .map(|value| value.access_key_id().to_string()),
             Some("cache-ak".to_string())
         );
     }
 
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn cloned_services_reuse_credentials_but_independent_services_do_not() {
+        let auth = AwsAuthService::default();
+        let config = AwsAuthConfig {
+            access_key_id: Some("configured-key".into()),
+            secret_access_key: Some("configured-secret".into()),
+            region_name: Some("us-east-1".into()),
+            ..AwsAuthConfig::default()
+        };
+        let flow = classify_auth(config.clone(), &no_env);
+        let cached = Credentials::new("cached-key", "cached-secret", None, None, "test");
+        auth.set_cached_credentials(
+            cache_key(&config, &flow),
+            cached.clone(),
+            STATIC_CREDENTIALS_TTL,
+        );
+
+        let reused = auth
+            .clone()
+            .resolve_credentials(config.clone(), &no_env)
+            .await
+            .unwrap();
+        let independent = AwsAuthService::default()
+            .resolve_credentials(config.clone(), &no_env)
+            .await
+            .unwrap();
+        let different = AwsAuthConfig {
+            access_key_id: Some("different-key".into()),
+            ..config.clone()
+        };
+        let other_identity = auth
+            .resolve_credentials(different.clone(), &no_env)
+            .await
+            .unwrap();
+
+        assert_eq!(reused.access_key_id(), cached.access_key_id());
+        assert_eq!(reused.secret_access_key(), cached.secret_access_key());
+        assert_eq!(
+            Some(independent.access_key_id()),
+            config.access_key_id.as_deref()
+        );
+        assert_eq!(
+            Some(other_identity.access_key_id()),
+            different.access_key_id.as_deref()
+        );
+    }
+
     #[test]
     fn same_role_comparison_matches_partition_account_and_role() {
         assert!(same_role_arns(
@@ -952,17 +1039,18 @@ mod tests {
         let body = br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":1,"messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}"#.to_vec();
         let headers =
             BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]);
-        let credentials = resolve_credentials(
+        let credentials = AwsAuthService::default()
-            AwsAuthConfig {
+            .resolve_credentials(
-                access_key_id: Some(access_key_id),
+                AwsAuthConfig {
-                secret_access_key: Some(secret_access_key),
+                    access_key_id: Some(access_key_id),
-                region_name: Some("us-west-2".to_string()),
+                    secret_access_key: Some(secret_access_key),
-                ..Default::default()
+                    region_name: Some("us-west-2".to_string()),
-            },
+                    ..Default::default()
-            &no_env,
+                },
-        )
+                &no_env,
-        .await?;
+            )
-        let client = reqwest::Client::new();
+            .await?;
+        let client = litellm_http::Client::plain_for_test();
         let mut failures = Vec::new();
 
         for region in ["us-west-2", "us-east-1"] {

litellm-rust/crates/auth-aws/src/signer.rs

@@ -1,13 +1,11 @@
 use std::{collections::BTreeMap, time::SystemTime};
 
+use crate::{
+    AwsAuthService, AwsCredentialSource, Error, aws_signature_headers, is_sigv4_computed_header,
+    sign_post,
+};
 use aws_credential_types::Credentials;
 use litellm_http::outbound::{RequestSigner, UnsignedRequest};
-use serde_json::{Map, Value};
-
-use crate::{
-    Error, aws_auth_config, aws_signature_headers, host_supplied_credentials,
-    is_sigv4_computed_header, resolve_credentials, sign_post,
-};
 
 #[derive(Clone, Debug)]
 pub struct SigV4Signer {
@@ -32,19 +30,17 @@ impl SigV4Signer {
     }
 
     pub async fn resolve(
+        auth: &AwsAuthService,
         region: String,
         service: &'static str,
-        optional_params: &Map<String, Value>,
+        credentials: AwsCredentialSource,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Self, Error> {
-        let credentials = match host_supplied_credentials(optional_params) {
+        Ok(Self::new(
-            Some(credentials) => credentials,
+            region,
-            None => {
+            service,
-                resolve_credentials(aws_auth_config(optional_params, env_lookup), env_lookup)
+            credentials.resolve(auth, env_lookup).await?,
-                    .await?
+        ))
-            }
-        };
-        Ok(Self::new(region, service, credentials))
     }
 }
 
@@ -80,7 +76,7 @@ mod tests {
     use std::time::{Duration, UNIX_EPOCH};
 
     use litellm_http::outbound::OutboundRequest;
-    use serde_json::json;
+    use serde_json::{Value, json};
 
     use super::*;
 

litellm-rust/crates/auth-azure/src/native.rs

@@ -133,7 +133,7 @@ impl NativeAzureTokenAcquirer {
         let token = credential
             .get_token(&[scope.as_str()], None)
             .await
-            .map_err(|error| Error::AzureTokenAcquisition(error.to_string()))?;
+            .map_err(|error| Error::CredentialAcquisition(error.to_string().into()))?;
         let expires_on = u64::try_from(token.expires_on.unix_timestamp())
             .ok()
             .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds));
@@ -250,7 +250,12 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> {
     let Some(authority) = authority else {
         return Ok(());
     };
-    let url = url::Url::parse(authority.value()).map_err(|_| Error::InvalidAzureAuthority)?;
+    let url = url::Url::parse(authority.value()).map_err(|_| {
+        Error::InvalidConfiguration(
+            "Azure authority must be an HTTPS origin without credentials, query, or fragment"
+                .into(),
+        )
+    })?;
     if url.scheme() != "https"
         || url.host_str().is_none()
         || !url.username().is_empty()
@@ -259,7 +264,10 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> {
         || url.fragment().is_some()
         || !matches!(url.path(), "" | "/")
     {
-        return Err(Error::InvalidAzureAuthority);
+        return Err(Error::InvalidConfiguration(
+            "Azure authority must be an HTTPS origin without credentials, query, or fragment"
+                .into(),
+        ));
     }
     Ok(())
 }
@@ -368,7 +376,9 @@ fn trusted_source(sources: &[InputSource]) -> InputSource {
 }
 
 fn mixed_sources<T>() -> Result<T, Error> {
-    Err(Error::MixedAzureCredentialSources)
+    Err(Error::InvalidConfiguration(
+        "request-controlled Azure auth inputs cannot be combined with host credentials".into(),
+    ))
 }
 
 fn build_credential(
@@ -433,7 +443,12 @@ fn build_credential(
         NativeAzureRequest::DeveloperTools { .. } => DeveloperToolsCredential::new(None)
             .map(|credential| credential as Arc<dyn TokenCredential>),
     }
-    .map_err(|error| Error::AzureCredentialInitialization(error.to_string()))
+    .map_err(|error| {
+        Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed(
+            "Azure credential initialization",
+            error,
+        ))
+    })
 }
 
 fn client_options(
@@ -638,7 +653,7 @@ mod tests {
         assert_eq!(transport.requests.lock().unwrap().len(), 6);
     }
 
-    #[test]
+    #[rstest::rstest]
     fn request_authority_requires_request_owned_client_secret_identity() {
         let error = ValidatedAzureRequest::new(sourced_client_secret(
             InputSource::Deployment,
@@ -647,10 +662,13 @@ mod tests {
         ))
         .unwrap_err();
 
-        assert!(matches!(
+        assert_eq!(
             error,
-            litellm_auth_types::Error::MixedAzureCredentialSources
+            litellm_auth_types::Error::InvalidConfiguration(
-        ));
+                "request-controlled Azure auth inputs cannot be combined with host credentials"
+                    .into()
+            )
+        );
     }
 
     #[test]
@@ -665,24 +683,24 @@ mod tests {
         assert_eq!(request.credential_source(), InputSource::Request);
     }
 
-    #[test]
+    #[rstest::rstest]
-    fn authority_is_restricted_to_an_https_origin() {
+    #[case::http("http://login.example")]
-        for authority in [
+    #[case::userinfo("https://user@login.example")]
-            "http://login.example",
+    #[case::path("https://login.example/tenant")]
-            "https://user@login.example",
+    #[case::query("https://login.example?target=other")]
-            "https://login.example/tenant",
+    fn authority_is_restricted_to_an_https_origin(#[case] authority: &str) {
-            "https://login.example?target=other",
+        let error = ValidatedAzureRequest::new(sourced_client_secret(
-        ] {
+            InputSource::Deployment,
-            let error = ValidatedAzureRequest::new(sourced_client_secret(
+            InputSource::Deployment,
-                InputSource::Deployment,
+            authority,
-                InputSource::Deployment,
+        ))
-                authority,
+        .unwrap_err();
-            ))
+        assert_eq!(
-            .unwrap_err();
+            error,
-            assert!(matches!(
+            litellm_auth_types::Error::InvalidConfiguration(
-                error,
+                "Azure authority must be an HTTPS origin without credentials, query, or fragment"
-                litellm_auth_types::Error::InvalidAzureAuthority
+                    .into()
-            ));
+            )
-        }
+        );
     }
 }

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -91,7 +91,9 @@ impl AzureAuthService {
             AzureCredentialPlan::Caller(caller) => {
                 let credential = caller.acquire().await?;
                 if credential.secret().expose().is_empty() {
-                    return Err(Error::EmptyAzureToken);
+                    return Err(Error::EmptyCallerCredential(
+                        "Azure AD token provider returned an empty token",
+                    ));
                 }
                 Ok(Some(Sourced::new(credential, InputSource::Deployment)))
             }
@@ -104,7 +106,11 @@ impl AzureAuthService {
             } => {
                 let assertion = resolve_reference(inputs, env_lookup, reference.value())
                     .await?
-                    .ok_or(Error::UnresolvedOidcReference)?;
+                    .ok_or_else(|| {
+                        Error::CredentialAcquisition(
+                            "Azure OIDC reference did not resolve to a value".into(),
+                        )
+                    })?;
                 let request = ValidatedAzureRequest::new(NativeAzureRequest::ClientAssertion {
                     tenant_id,
                     client_id,
@@ -167,7 +173,7 @@ pub(crate) fn select_auth_plan(
                 .map(|selector| Sourced::new(selector, value.source()))
         })
         .transpose()
-        .map_err(|_| Error::InvalidAzureSelector)?;
+        .map_err(|_| Error::InvalidConfiguration("invalid Azure credential selector".into()))?;
     let federated_token_file = configured_string(
         &inputs.federated_token_file,
         AZURE_FEDERATED_TOKEN_FILE_ENV,
@@ -257,7 +263,9 @@ fn select_native_plan(
     let selection_source = selected.source();
 
     match selected.into_value() {
-        AzureCredentialType::ClientSecretCredential => Err(Error::MissingClientSecretFields),
+        AzureCredentialType::ClientSecretCredential => Err(Error::InvalidConfiguration(
+            "ClientSecretCredential requires tenant_id, client_id, and client_secret".into(),
+        )),
         AzureCredentialType::WorkloadIdentityCredential => {
             Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new(
                 workload_request(tenant_id, client_id, federated_token_file, scope, authority)?,
@@ -341,9 +349,17 @@ fn workload_request(
     authority: Option<Sourced<String>>,
 ) -> Result<NativeAzureRequest, Error> {
     Ok(NativeAzureRequest::WorkloadIdentity {
-        tenant_id: tenant_id.ok_or(Error::MissingWorkloadTenant)?,
+        tenant_id: tenant_id.ok_or_else(|| {
-        client_id: client_id.ok_or(Error::MissingWorkloadClient)?,
+            Error::InvalidConfiguration("WorkloadIdentityCredential requires tenant_id".into())
-        token_file_path: token_file_path.ok_or(Error::MissingWorkloadTokenFile)?,
+        })?,
+        client_id: client_id.ok_or_else(|| {
+            Error::InvalidConfiguration("WorkloadIdentityCredential requires client_id".into())
+        })?,
+        token_file_path: token_file_path.ok_or_else(|| {
+            Error::InvalidConfiguration(
+                "WorkloadIdentityCredential requires azure_federated_token_file".into(),
+            )
+        })?,
         scope,
         authority,
     })
@@ -394,10 +410,11 @@ async fn resolve_reference(
             .map_or(CredentialLookup::Missing, CredentialLookup::Found),
         CredentialRef::None => return Ok(None),
         CredentialRef::File(_) | CredentialRef::Request(_) | CredentialRef::Host(_) => {
-            let resolver = inputs
+            let resolver = inputs.credential_resolver.as_ref().ok_or_else(|| {
-                .credential_resolver
+                Error::InvalidConfiguration(
-                .as_ref()
+                    "credential reference requires a host credential resolver".into(),
-                .ok_or(Error::MissingHostResolver)?;
+                )
+            })?;
             resolver.resolve(reference).await?
         }
     };
@@ -415,7 +432,9 @@ fn oidc_reference(
     };
     let value = token.value().expose();
     if token.source() == InputSource::Request && value.starts_with("oidc/") {
-        return Err(Error::RequestAzureCredentialReference);
+        return Err(Error::InvalidConfiguration(
+            "request-controlled Azure credential references are not allowed".into(),
+        ));
     }
     if let Some(name) = value.strip_prefix("oidc/env/") {
         return non_empty_reference(name, "OIDC environment reference")
@@ -437,14 +456,20 @@ fn oidc_reference(
         )));
     }
     if value.starts_with("oidc/") {
-        return Err(Error::UnsupportedOidcReference);
+        return Err(Error::InvalidConfiguration(
+            "unsupported OIDC reference".into(),
+        ));
     }
     Ok(None)
 }
 
 fn non_empty_reference(value: &str, kind: &str) -> Result<String, Error> {
     if value.is_empty() {
-        return Err(Error::EmptyReference(kind.to_string()));
+        return Err(Error::InvalidConfiguration(
+            litellm_auth_types::ErrorDetail::Empty {
+                subject: kind.into(),
+            },
+        ));
     }
     Ok(value.to_string())
 }
@@ -493,7 +518,7 @@ mod tests {
                         expires_on: None,
                     })
                 } else {
-                    Err(Error::AzureTokenAcquisition(format!("{kind} failed")))
+                    Err(Error::CredentialAcquisition(kind.into()))
                 }
             })
         }
@@ -602,7 +627,7 @@ mod tests {
         assert!(error.to_string().contains("unsupported OIDC reference"));
     }
 
-    #[test]
+    #[rstest::rstest]
     fn request_oidc_reference_is_rejected_before_lookup() {
         let params = json!({
             "azure_ad_token": "oidc/env/ASSERTION",
@@ -624,7 +649,12 @@ mod tests {
         })
         .unwrap_err();
 
-        assert!(matches!(error, Error::RequestAzureCredentialReference));
+        assert_eq!(
+            error,
+            Error::InvalidConfiguration(
+                "request-controlled Azure credential references are not allowed".into()
+            )
+        );
     }
 
     #[tokio::test]
@@ -723,6 +753,7 @@ mod tests {
         assert_eq!(credential.value().secret().expose(), "caller-token");
     }
 
+    #[rstest::rstest]
     #[tokio::test]
     async fn empty_caller_token_is_rejected() {
         let error = AzureAuthService::default()
@@ -730,6 +761,9 @@ mod tests {
             .await
             .unwrap_err();
 
-        assert!(matches!(error, Error::EmptyAzureToken));
+        assert_eq!(
+            error,
+            Error::EmptyCallerCredential("Azure AD token provider returned an empty token")
+        );
     }
 }

litellm-rust/crates/auth-azure/src/types.rs

@@ -117,7 +117,12 @@ fn string_config(
         None => Ok(ConfigValue::Absent),
         Some(Value::Null) => Ok(ConfigValue::ExplicitNone(source)),
         Some(Value::String(value)) => Ok(ConfigValue::Value(Sourced::new(value.clone(), source))),
-        Some(_) => Err(Error::InvalidFieldType(name.to_string())),
+        Some(_) => Err(Error::InvalidConfiguration(
+            litellm_auth_types::ErrorDetail::InvalidType {
+                field: name.into(),
+                expected: "a string or null",
+            },
+        )),
     }
 }
 

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -19,3 +19,6 @@ tokio.workspace = true
 gcp_auth = "0.12.7"
 google-cloud-auth = { workspace = true, optional = true }
 http = { workspace = true, optional = true }
+
+[dev-dependencies]
+rstest.workspace = true

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -131,7 +131,7 @@ impl Default for VertexAuth {
 }
 
 impl VertexAuth {
-    fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
+    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
         Self {
             providers: Cache::builder().max_capacity(64).build(),
             loader,
@@ -220,16 +220,16 @@ impl VertexAuth {
     }
 }
 
-trait VertexTokenSource: Send + Sync {
+pub trait VertexTokenSource: Send + Sync {
     fn project_id(&self) -> VertexAuthFuture<'_, String>;
     fn token(&self) -> VertexAuthFuture<'_, String>;
 }
 
-trait VertexProviderLoader: Send + Sync {
+pub trait VertexProviderLoader: Send + Sync {
     fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
 }
 
-type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
 
 struct GcpTokenSource(Arc<dyn TokenProvider>);
 
@@ -299,13 +299,13 @@ fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
                 .map(str::to_string)
         });
     if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
-        return Err(Error::RequestVertexTokenEndpoint);
+        return Err(Error::InvalidConfiguration("request-controlled Vertex credentials must use the canonical Google OAuth token endpoint".into()));
     }
     Ok(configured)
 }
 
 #[derive(Clone, Debug)]
-enum CredentialSource {
+pub enum CredentialSource {
     Inline(SecretValue),
     Trusted(SecretValue),
     ApplicationCredentials(String),
@@ -376,10 +376,20 @@ fn optional_credentials(
                     .map(SecretValue::new)
                     .map(|value| Sourced::new(value, source))
                     .map(Some)
-                    .map_err(|error| Error::InvalidFieldType(format!("{}: {error}", names[0])));
+                    .map_err(|error| {
+                        Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed(
+                            "credential serialization",
+                            error,
+                        ))
+                    });
             }
             Some(_) => {
-                return Err(Error::InvalidFieldType(names[0].to_string()));
+                return Err(Error::InvalidConfiguration(
+                    litellm_auth_types::ErrorDetail::InvalidType {
+                        field: names[0].into(),
+                        expected: "a string or null",
+                    },
+                ));
             }
         }
     }
@@ -397,7 +407,12 @@ fn optional_string(params: &Map<String, Value>, names: &[&str]) -> Result<Option
             Some(Value::String(value)) if value.trim().is_empty() => continue,
             Some(Value::String(value)) => return Ok(Some(value.clone())),
             Some(_) => {
-                return Err(Error::InvalidFieldType(names[0].to_string()));
+                return Err(Error::InvalidConfiguration(
+                    litellm_auth_types::ErrorDetail::InvalidType {
+                        field: names[0].into(),
+                        expected: "a string or null",
+                    },
+                ));
             }
         }
     }
@@ -411,7 +426,10 @@ fn non_empty_env(env_lookup: &dyn Fn(&str) -> Option<String>, name: &str) -> Opt
 }
 
 fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
-    Error::VertexTokenAcquisition(error.to_string())
+    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
+        "Vertex AI credentials",
+        error,
+    ))
 }
 
 #[cfg(test)]
@@ -612,20 +630,15 @@ mod tests {
         );
     }
 
-    #[test]
+    #[rstest::rstest]
-    fn request_credentials_require_canonical_token_endpoint() {
+    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
-        assert!(
+    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
-            validate_request_credentials(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#)
+    #[case::missing_endpoint("{}", false)]
-                .is_ok()
+    fn request_credentials_require_canonical_token_endpoint(
-        );
+        #[case] credentials: &str,
-        assert!(matches!(
+        #[case] accepted: bool,
-            validate_request_credentials(r#"{"token_uri":"http://127.0.0.1/token"}"#),
+    ) {
-            Err(Error::RequestVertexTokenEndpoint)
+        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
-        ));
-        assert!(matches!(
-            validate_request_credentials("{}"),
-            Err(Error::RequestVertexTokenEndpoint)
-        ));
     }
 
     #[tokio::test]

litellm-rust/crates/auth-types/Cargo.toml

@@ -12,4 +12,5 @@ thiserror.workspace = true
 veil.workspace = true
 
 [dev-dependencies]
+rstest.workspace = true
 tokio.workspace = true

litellm-rust/crates/auth-types/src/credential.rs

@@ -86,7 +86,9 @@ impl CredentialPlan {
             Self::Caller(caller) => {
                 let credential = caller.acquire().await?;
                 if credential.secret().expose().is_empty() {
-                    return Err(Error::EmptyCallerCredential);
+                    return Err(Error::EmptyCallerCredential(
+                        "credential caller returned an empty credential",
+                    ));
                 }
                 Ok(CredentialPlanResolution::Resolved(credential))
             }
@@ -147,10 +149,15 @@ mod tests {
 
     impl CredentialResolver for FailingResolver {
         fn resolve<'a>(&'a self, _reference: &'a CredentialRef) -> CredentialLookupFuture<'a> {
-            Box::pin(async { Err(Error::UnresolvedOidcReference) })
+            Box::pin(async {
+                Err(Error::CredentialAcquisition(
+                    "host credential lookup failed".into(),
+                ))
+            })
         }
     }
 
+    #[rstest::rstest]
     #[tokio::test]
     async fn acquisition_failure_is_terminal() {
         let resolver = CredentialResolverHandle::new(Arc::new(FailingResolver));
@@ -161,6 +168,9 @@ mod tests {
             .await
             .expect_err("acquisition errors cannot become fallback");
 
-        assert_eq!(error, Error::UnresolvedOidcReference);
+        assert_eq!(
+            error,
+            Error::CredentialAcquisition("host credential lookup failed".into())
+        );
     }
 }

litellm-rust/crates/auth-types/src/error.rs

@@ -2,84 +2,16 @@ use thiserror::Error as ThisError;
 
 #[derive(Clone, Debug, ThisError, PartialEq, Eq)]
 pub enum Error {
-    #[error("invalid authentication configuration: credential header already exists")]
+    #[error("invalid authentication configuration: {0}")]
-    ExistingCredentialHeader,
+    InvalidConfiguration(#[source] ErrorDetail),
-    #[error(
-        "invalid authentication configuration: credential plan is not allowed by the provider auth policy"
-    )]
-    DisallowedCredentialPlan,
-    #[error("invalid authentication configuration: credential cannot be empty")]
-    EmptyCredential,
-    #[error("invalid authentication configuration: invalid Azure credential selector")]
-    InvalidAzureSelector,
-    #[error(
-        "invalid authentication configuration: ClientSecretCredential requires tenant_id, client_id, and client_secret"
-    )]
-    MissingClientSecretFields,
-    #[error("invalid authentication configuration: WorkloadIdentityCredential requires tenant_id")]
-    MissingWorkloadTenant,
-    #[error("invalid authentication configuration: WorkloadIdentityCredential requires client_id")]
-    MissingWorkloadClient,
-    #[error(
-        "invalid authentication configuration: WorkloadIdentityCredential requires azure_federated_token_file"
-    )]
-    MissingWorkloadTokenFile,
-    #[error(
-        "invalid authentication configuration: credential reference requires a host credential resolver"
-    )]
-    MissingHostResolver,
-    #[error(
-        "invalid authentication configuration: caller credential plan requires provider-specific inputs"
-    )]
-    MissingCallerInputs,
-    #[error("invalid authentication configuration: credential header {0} already exists")]
-    DuplicateHeader(&'static str),
-    #[error("invalid authentication configuration: {0} must be a string or null")]
-    InvalidFieldType(String),
-    #[error("invalid authentication configuration: unsupported OIDC reference")]
-    UnsupportedOidcReference,
-    #[error("invalid authentication configuration: {0} cannot be empty")]
-    EmptyReference(String),
-    #[error("invalid authentication configuration: Azure credential initialization failed: {0}")]
-    AzureCredentialInitialization(String),
-    #[error(
-        "invalid authentication configuration: Azure authority must be an HTTPS origin without credentials, query, or fragment"
-    )]
-    InvalidAzureAuthority,
-    #[error(
-        "invalid authentication configuration: request-controlled Azure auth inputs cannot be combined with host credentials"
-    )]
-    MixedAzureCredentialSources,
-    #[error(
-        "invalid authentication configuration: request-controlled Azure credential references are not allowed"
-    )]
-    RequestAzureCredentialReference,
-    #[error(
-        "invalid authentication configuration: host credentials cannot be sent to a request-controlled Azure endpoint"
-    )]
-    RequestAzureCredentialDestination,
-    #[error(
-        "invalid authentication configuration: credentials cannot be sent to a request-controlled Vertex AI endpoint"
-    )]
-    RequestVertexCredentialDestination,
-    #[error(
-        "invalid authentication configuration: request-controlled Vertex credentials must use the canonical Google OAuth token endpoint"
-    )]
-    RequestVertexTokenEndpoint,
     #[error("credential acquisition failed: {0}")]
-    AzureTokenAcquisition(String),
+    CredentialAcquisition(#[source] ErrorDetail),
-    #[error("credential acquisition failed: Vertex AI credentials: {0}")]
+    #[error("credential caller failed: {0}")]
-    VertexTokenAcquisition(String),
+    EmptyCallerCredential(&'static str),
     #[error("{0}")]
     ProviderAuthentication(String),
     #[error("credential acquisition failed: {}", .0.iter().map(ToString::to_string).collect::<Vec<_>>().join("; "))]
     CredentialChain(Vec<Error>),
-    #[error("credential caller failed: credential caller returned an empty credential")]
-    EmptyCallerCredential,
-    #[error("credential caller failed: Azure AD token provider returned an empty token")]
-    EmptyAzureToken,
-    #[error("credential acquisition failed: Azure OIDC reference did not resolve to a value")]
-    UnresolvedOidcReference,
     #[error(
         "Missing {provider} API Key - Set `api_key` or the {environment_variable} environment variable"
     )]
@@ -87,34 +19,87 @@ pub enum Error {
         provider: &'static str,
         environment_variable: &'static str,
     },
-    #[error(
+    #[error("Missing {provider} API Base - {guidance}")]
-        "Missing {provider} API Base - Set {environment_variable} environment variable or pass api_base parameter"
-    )]
     MissingApiBase {
         provider: &'static str,
-        environment_variable: &'static str,
+        guidance: &'static str,
     },
-    #[error(
-        "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
-    )]
-    MissingAzureApiBase,
     #[error("invalid authentication header")]
     InvalidHeader,
 }
 
-#[cfg(test)]
+#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
-mod tests {
+pub enum ErrorDetail {
-    use super::Error;
+    #[error("{0}")]
+    Message(String),
+    #[error("{field} must be {expected}")]
+    InvalidType {
+        field: String,
+        expected: &'static str,
+    },
+    #[error("{subject} cannot be empty")]
+    Empty { subject: String },
+    #[error("credential header {0} already exists")]
+    DuplicateHeader(&'static str),
+    #[error("{operation} failed: {source}")]
+    Failed {
+        operation: &'static str,
+        #[source]
+        source: ErrorSource,
+    },
+}
 
-    #[test]
+impl ErrorDetail {
-    fn missing_api_key_names_provider_and_environment_variable() {
+    pub fn failed(
-        assert_eq!(
+        operation: &'static str,
-            Error::MissingApiKey {
+        source: impl std::error::Error + Send + Sync + 'static,
-                provider: "Anthropic",
+    ) -> Self {
-                environment_variable: "ANTHROPIC_API_KEY",
+        Self::Failed {
-            }
+            operation,
-            .to_string(),
+            source: ErrorSource::new(source),
-            "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable"
+        }
-        );
     }
 }
+
+impl From<String> for ErrorDetail {
+    fn from(message: String) -> Self {
+        Self::Message(message)
+    }
+}
+
+impl From<&str> for ErrorDetail {
+    fn from(message: &str) -> Self {
+        Self::Message(message.into())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct ErrorSource(std::sync::Arc<dyn std::error::Error + Send + Sync>);
+
+impl std::ops::Deref for ErrorSource {
+    type Target = dyn std::error::Error + Send + Sync;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref()
+    }
+}
+
+impl std::fmt::Display for ErrorSource {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.0, formatter)
+    }
+}
+
+impl ErrorSource {
+    pub fn new(error: impl std::error::Error + Send + Sync + 'static) -> Self {
+        Self(std::sync::Arc::new(error))
+    }
+}
+
+impl PartialEq for ErrorSource {
+    fn eq(&self, other: &Self) -> bool {
+        std::sync::Arc::ptr_eq(&self.0, &other.0)
+    }
+}
+
+impl Eq for ErrorSource {}

litellm-rust/crates/auth-types/src/http.rs

@@ -7,7 +7,7 @@ pub enum CredentialPlacement {
 }
 
 impl CredentialPlacement {
-    pub fn header_name(self) -> &'static str {
+    pub const fn header_name(self) -> &'static str {
         match self {
             Self::Bearer => "Authorization",
             Self::Header(name) => name,
@@ -21,13 +21,17 @@ pub fn apply_credential(
     placement: CredentialPlacement,
 ) -> Result<Vec<(String, String)>, Error> {
     if credential.trim().is_empty() {
-        return Err(Error::EmptyCredential);
+        return Err(Error::InvalidConfiguration(
+            "credential cannot be empty".into(),
+        ));
     }
     if headers
         .iter()
         .any(|(name, _)| name.eq_ignore_ascii_case(placement.header_name()))
     {
-        return Err(Error::DuplicateHeader(placement.header_name()));
+        return Err(Error::InvalidConfiguration(
+            crate::ErrorDetail::DuplicateHeader(placement.header_name()),
+        ));
     }
     let value = match placement {
         CredentialPlacement::Bearer => format!("Bearer {credential}"),
@@ -40,21 +44,6 @@ pub fn apply_credential(
     )
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum RequestAuth {
-    Header {
-        name: &'static str,
-        value: String,
-    },
-    Bearer {
-        token: String,
-    },
-    AwsSigV4 {
-        region: String,
-        service: &'static str,
-    },
-}
-
 #[cfg(test)]
 mod tests {
     use super::{CredentialPlacement, apply_credential};