Merge remote-tracking branch 'origin/main' into HEAD

.circleci/config.yml

@@ -12,6 +12,9 @@ parameters:
   migration_source_sha:
     type: string
     default: ""
+  routing_parity_base:
+    type: string
+    default: ""
 orbs:
   codecov: codecov/codecov@4.0.1
   node: circleci/node@5.1.0 # Add this line to declare the node orb
@@ -138,7 +141,7 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
     steps:
       - run:
           name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
@@ -164,9 +167,29 @@ commands:
             /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
             rustc --version
             cargo --version
+            { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
+  save_cargo_target:
+    steps:
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/debug
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -176,6 +199,9 @@ commands:
       image:
         type: string
         default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+      server_args:
+        type: string
+        default: ""
     steps:
       - run:
           name: Start PostgreSQL
@@ -186,7 +212,7 @@ commands:
               -e POSTGRES_PASSWORD=postgres \
               -e POSTGRES_DB=<< parameters.db_name >> \
               -p 5432:5432 \
-              << parameters.image >>
+              << parameters.image >> << parameters.server_args >>
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
@@ -275,50 +301,11 @@ commands:
             # `uv sync --package litellm-enterprise` here — that overwrites the
             # shared .venv and strips out dev/test deps (pytest, prisma, etc.).
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
-  setup_litellm_test_deps:
+  install_windows_toolchain:
     steps:
-      - checkout
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
       - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
-
-jobs:
-  # Add Windows testing job
-  using_litellm_on_windows:
-    executor:
-      name: win/default
-      shell: powershell.exe
-    working_directory: ~/project
-    environment:
-      UV_PYTHON: "3.11"
-      CARGO_HTTP_MULTIPLEXING: "false"
-      CARGO_NET_RETRY: "5"
-    steps:
-      - checkout
-      - run:
-          name: Install Python
-          command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
-      - run:
-          name: Install Dependencies
-          environment:
-            UV_HTTP_TIMEOUT: "300"
+          name: Install Rust and uv
+          no_output_timeout: 30m
           command: |
             $rustupInit = Join-Path $env:TEMP "rustup-init.exe"
             $rustupVersion = "1.28.2"
@@ -358,6 +345,55 @@ jobs:
             if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
+  setup_litellm_test_deps:
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
+
+jobs:
+  # Add Windows testing job
+  using_litellm_on_windows:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Install Dependencies
+          no_output_timeout: 30m
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             for ($attempt = 1; $attempt -le 5; $attempt++) {
               Write-Host "uv sync attempt $attempt/5"
               uv sync --frozen --group dev --python 3.11
@@ -373,16 +409,68 @@ jobs:
           name: Run Windows-specific test
           command: |
             uv run --no-sync python -m pytest tests/windows_tests/ -v
+
+  windows_release_wheel:
+    executor:
+      name: win/default
+      shell: powershell.exe
+      size: xlarge
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
       - run:
-          name: Guard against MAX_PATH-busting packaged wheel paths
+          name: Skip job when no windows-release-relevant files changed
+          shell: bash.exe
+          command: bash .circleci/scripts/path_filter.sh windows-release
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Record the Rust build environment for the release cargo cache key
+          command: |
+            & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: |
+            $fingerprints = "litellm-rust/target/release/.fingerprint"
+            if (Test-Path $fingerprints) {
+              Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
+            }
+      - run:
+          name: Build the release wheel and install it under a worst-case MAX_PATH prefix
+          no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
             $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
-            cargo --version
-            Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
             uv build --wheel --out-dir dist
-            uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
+            if ($LASTEXITCODE -ne 0) {
+              exit $LASTEXITCODE
+            }
+            python tests/windows_tests/check_windows_wheel_install.py
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/release
 
   base_sdk_install:
     docker:
@@ -410,6 +498,10 @@ jobs:
             uv venv /tmp/base-sdk --python 3.12
             VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
             /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+      - run:
+          name: Guard against MAX_PATH-busting packaged wheel paths
+          command: |
+            python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
 
   local_testing_part1:
     docker:
@@ -438,6 +530,7 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -3108,10 +3201,18 @@ jobs:
     parameters:
       suite:
         type: string
+      mode:
+        type: enum
+        enum: [standard, replica]
+        default: standard
+      parallelism:
+        type: integer
+        default: 1
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: << parameters.parallelism >>
     steps:
       - setup_litellm_test_deps
       - when:
@@ -3142,18 +3243,19 @@ jobs:
                 command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
       - start_redis
       - run:
           name: Run owned integration contracts
-          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >>
           no_output_timeout: 15m
       - run:
           name: Stop owned database and Redis
           when: always
           command: |
-            mkdir -p test-results/integration-<< parameters.suite >>
-            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
-            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
+            mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >>
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true
             docker rm -f postgres-db redis-cache
             test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
       - store_test_results:
@@ -3161,11 +3263,82 @@ jobs:
       - store_artifacts:
           path: test-results
 
+  routing_parity:
+    parameters:
+      suite:
+        type: string
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Check out base product code
+          environment:
+            ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
+          command: |
+            [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
+            git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
+            git rm -r -f --quiet litellm enterprise litellm-proxy-extras
+            git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
+            git reset --quiet
+            test -f litellm/rust_bridge/_native.abi3.so
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
+      - start_redis
+      - run:
+          name: Run base side
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base
+          no_output_timeout: 15m
+      - run:
+          name: Stop base database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/services-<< parameters.suite >>-parity-base
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - run:
+          name: Check out head product code
+          command: |
+            git rm -r -f --quiet litellm enterprise litellm-proxy-extras
+            git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras
+            git reset --quiet
+            test -f litellm/rust_bridge/_native.abi3.so
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
+      - start_redis
+      - run:
+          name: Run head side
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head
+          no_output_timeout: 15m
+      - run:
+          name: Stop head database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/services-<< parameters.suite >>-parity-head
+            docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - run:
+          name: Compare routing parity
+          command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
   unit:
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: 4
     steps:
       - setup_litellm_test_deps
       - run:
@@ -3175,10 +3348,11 @@ jobs:
           name: Run unit tests
           command: |
             mkdir -p test-results/unit
-            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
-            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
+            if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
+            mapfile -t files < <(printf '%s\n' "${shard}")
             set +e
-            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
@@ -3224,117 +3398,100 @@ workflows:
             branches:
               only: main
     jobs: *migration_jobs
+  routing_parity:
+    when:
+      not:
+        equal: ["", << pipeline.parameters.routing_parity_base >>]
+    jobs:
+      - routing_parity:
+          name: routing-parity-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [management, accounting, database, providers, extensions, cost, mcp]
   integration:
-    unless: << pipeline.parameters.run_migration_tests >>
+    unless:
+      or:
+        - << pipeline.parameters.run_migration_tests >>
+        - not:
+            equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
       - integration_contracts:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, extensions, mcp, sdk, cost, browser]
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
+              suite: [management, accounting, database, providers, mcp, sdk, cost, browser]
+      - integration_contracts:
+          name: integration-extensions
+          suite: extensions
+          parallelism: 4
+      - integration_contracts:
+          name: integration-<< matrix.suite >>-replica
+          matrix:
+            parameters:
+              suite: [management, database]
+              mode: [replica]
   build_and_test:
-    unless: << pipeline.parameters.run_migration_tests >>
+    unless:
+      or:
+        - << pipeline.parameters.run_migration_tests >>
+        - not:
+            equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
-      - using_litellm_on_windows:
-          filters: &main_branches
-            branches:
-              only:
-                - main
-                - /litellm_.*/
-      - unit:
-          filters: *main_branches
+      - using_litellm_on_windows
+      - windows_release_wheel
+      - unit
       - provider_replay_harness
-      - base_sdk_install:
-          filters: *main_branches
-      - local_testing_part1:
-          filters: *main_branches
-      - local_testing_part2:
-          filters: *main_branches
-      - langfuse_logging_unit_tests:
-          filters: *main_branches
-      - litellm_assistants_api_testing:
-          filters: *main_branches
-      - litellm_router_testing:
-          filters: *main_branches
-      - litellm_router_unit_testing:
-          filters: *main_branches
-      - auth_ui_unit_tests:
-          filters: *main_branches
-      - build_docker_database_image:
-          filters: *main_branches
-      - e2e_ui_testing:
-          filters: *main_branches
-      - e2e_ui_testing_server_root_path:
-          filters: *main_branches
+      - base_sdk_install
+      - local_testing_part1
+      - local_testing_part2
+      - langfuse_logging_unit_tests
+      - litellm_assistants_api_testing
+      - litellm_router_testing
+      - litellm_router_unit_testing
+      - auth_ui_unit_tests
+      - build_docker_database_image
+      - e2e_ui_testing
+      - e2e_ui_testing_server_root_path
       - build_and_test:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - e2e_openai_endpoints:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_logging_guardrails_model_info_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_spend_accuracy_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_multi_instance_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_store_model_in_db_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - proxy_build_from_pip_tests:
-          filters: *main_branches
+      - proxy_build_from_pip_tests
       - proxy_pass_through_endpoint_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - proxy_e2e_anthropic_messages_tests:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - llm_translation_testing:
-          filters: *main_branches
-      - realtime_translation_testing:
-          filters: *main_branches
-      - agent_testing:
-          filters: *main_branches
-      - guardrails_testing:
-          filters: *main_branches
-      - google_generate_content_endpoint_testing:
-          filters: *main_branches
-      - llm_responses_api_testing:
-          filters: *main_branches
-      - ocr_testing:
-          filters: *main_branches
-      - search_testing:
-          filters: *main_branches
-      - batches_testing:
-          filters: *main_branches
-      - litellm_utils_testing:
-          filters: *main_branches
-      - pass_through_unit_testing:
-          filters: *main_branches
-      - image_gen_testing:
-          filters: *main_branches
-      - logging_testing:
-          filters: *main_branches
-      - audio_testing:
-          filters: *main_branches
-      - redis_caching_unit_tests:
-          filters: *main_branches
+      - llm_translation_testing
+      - realtime_translation_testing
+      - agent_testing
+      - guardrails_testing
+      - google_generate_content_endpoint_testing
+      - llm_responses_api_testing
+      - ocr_testing
+      - search_testing
+      - batches_testing
+      - litellm_utils_testing
+      - pass_through_unit_testing
+      - image_gen_testing
+      - logging_testing
+      - audio_testing
+      - redis_caching_unit_tests
       - upload-coverage:
           requires:
             - realtime_translation_testing
@@ -3359,18 +3516,12 @@ workflows:
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-          filters: *main_branches
-      - installing_litellm_on_python:
-          filters: *main_branches
-      - installing_litellm_on_python_3_13:
-          filters: *main_branches
-      - installing_litellm_on_python_v2_migration_resolver:
-          filters: *main_branches
+      - installing_litellm_on_python
+      - installing_litellm_on_python_3_13
+      - installing_litellm_on_python_v2_migration_resolver
       - helm_chart_testing:
           requires:
             - build_docker_database_image
-          filters: *main_branches
       - test_bad_database_url:
           requires:
             - build_docker_database_image
-          filters: *main_branches

.circleci/scripts/classify_changes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
 
 has_client=false
 has_backend=false
@@ -9,19 +9,24 @@ has_ci=false
 has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
+has_windows_release=false
 outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
     *.md | *.mdx) : ;;
-    pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/test_litellm/test_circleci_path_filter.py | tests/test_litellm/test_detect_changes.py)
+    pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
       has_mcp_dependencies=true ;;
   esac
   case "$file" in
     tests/e2e/*/*.py) : ;;
-    tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
+    tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
       has_provider_harness=true ;;
   esac
+  case "$file" in
+    litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
+      has_windows_release=true ;;
+  esac
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
@@ -31,7 +36,7 @@ while IFS= read -r file || [ -n "$file" ]; do
   case "$file" in
     model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
       has_cost_map=true ;;
-    tests/test_litellm/* | tests/proxy_unit_tests/*) : ;;
+    tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;;
     *) outside_cost_map_set=true ;;
   esac
 done
@@ -46,6 +51,9 @@ case "$category" in
   provider-harness)
     [ "$has_provider_harness" = true ] && echo run || echo skip
     ;;
+  windows-release)
+    [ "$has_windows_release" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/prepare_replica_roles.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+from typing import Final
+from urllib.parse import urlsplit, urlunsplit
+
+import psycopg
+
+DATABASE_URL: Final = os.environ["DATABASE_URL"]
+
+
+def postgres_url() -> str:
+    parsed: Final = urlsplit(DATABASE_URL)
+    return urlunsplit(parsed._replace(path="/postgres"))
+
+
+def main() -> None:
+    with psycopg.connect(postgres_url(), autocommit=True) as admin:
+        admin.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
+        admin.execute("CREATE ROLE litellm_writer LOGIN PASSWORD 'litellm-writer' NOSUPERUSER")
+        admin.execute("CREATE ROLE litellm_reader LOGIN PASSWORD 'litellm-reader' NOSUPERUSER NOINHERIT")
+        admin.execute("ALTER ROLE litellm_reader SET default_transaction_read_only = on")
+        admin.execute("ALTER DATABASE circle_test OWNER TO litellm_writer")
+        admin.execute("GRANT CONNECT ON DATABASE circle_test TO litellm_reader")
+    with psycopg.connect(DATABASE_URL, autocommit=True) as admin:
+        admin.execute("GRANT USAGE ON SCHEMA public TO litellm_reader")
+        admin.execute(
+            "ALTER DEFAULT PRIVILEGES FOR ROLE litellm_writer IN SCHEMA public GRANT SELECT ON TABLES TO litellm_reader"
+        )
+        admin.execute("GRANT SELECT ON ALL TABLES IN SCHEMA public TO litellm_reader")
+
+    parsed: Final = urlsplit(DATABASE_URL)
+    reader_url: Final = urlunsplit(
+        parsed._replace(netloc=f"litellm_reader:litellm-reader@{parsed.hostname}:{parsed.port}")
+    )
+    writer_url: Final = urlunsplit(
+        parsed._replace(netloc=f"litellm_writer:litellm-writer@{parsed.hostname}:{parsed.port}")
+    )
+    with psycopg.connect(reader_url, autocommit=True) as reader:
+        assert reader.execute("SHOW transaction_read_only").fetchone() == ("on",)
+        try:
+            reader.execute("CREATE TABLE integration_readonly_probe (id int)")
+        except psycopg.errors.ReadOnlySqlTransaction:
+            pass
+        else:
+            raise AssertionError("litellm_reader executed a write statement")
+    with psycopg.connect(writer_url, autocommit=True) as writer:
+        assert writer.execute("SELECT current_user").fetchone() == ("litellm_writer",)
+
+
+if __name__ == "__main__":
+    main()

.circleci/scripts/run_integration.sh

@@ -7,7 +7,15 @@ if [ "${GITHUB_ACTIONS:-}" = true ]; then
 fi
 
 suite="${1:?integration suite required}"
-results="test-results/integration-${suite}"
+mode="${2:-standard}"
+side="${3:-}"
+if [ "$mode" = replica ]; then
+  results="test-results/integration-${suite}-replica"
+elif [ "$mode" = parity ]; then
+  results="test-results/parity-${suite}/${side:?parity side required}"
+else
+  results="test-results/integration-${suite}"
+fi
 mkdir -p "$results"
 integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')"
 upstream_pid=""
@@ -18,6 +26,7 @@ guard_created=false
 guard_installed=false
 guard6_created=false
 guard6_installed=false
+egress_cgroup=litellm-integration
 cleanup() {
   original_status=$?
   trap - EXIT INT TERM
@@ -39,14 +48,14 @@ cleanup() {
     fi
   done
   if [ "$guard_installed" = true ]; then
-    sudo iptables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo iptables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard_created" = true ]; then
     sudo iptables -F integration_only || original_status=1
     sudo iptables -X integration_only || original_status=1
   fi
   if [ "$guard6_installed" = true ]; then
-    sudo ip6tables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+    sudo ip6tables -D OUTPUT -m cgroup --path "$egress_cgroup" -j integration_only || original_status=1
   fi
   if [ "$guard6_created" = true ]; then
     sudo ip6tables -F integration_only || original_status=1
@@ -80,6 +89,20 @@ export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
 
 uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
 
+export INTEGRATION_PROXY_DATABASE_URL=""
+export INTEGRATION_PROXY_READ_REPLICA_URL=""
+export INTEGRATION_ROUTING=""
+if [ "$mode" = replica ] || [ "$mode" = parity ]; then
+  .venv/bin/python .circleci/scripts/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1
+  export INTEGRATION_PROXY_DATABASE_URL="postgresql://litellm_writer:litellm-writer@127.0.0.1:5432/circle_test"
+  export INTEGRATION_PROXY_READ_REPLICA_URL="postgresql://litellm_reader:litellm-reader@127.0.0.1:5432/circle_test"
+fi
+if [ "$mode" = parity ]; then
+  export INTEGRATION_ROUTING=capture
+fi
+
+sudo mkdir -p "/sys/fs/cgroup/$egress_cgroup"
+echo "$$" | sudo tee "/sys/fs/cgroup/$egress_cgroup/cgroup.procs" > /dev/null
 sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
@@ -89,13 +112,13 @@ for service in postgres-db redis-cache; do
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
 sudo iptables -A integration_only -j REJECT
-sudo iptables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo iptables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard_installed=true
 sudo ip6tables -N integration_only
 guard6_created=true
 sudo ip6tables -A integration_only -o lo -j ACCEPT
 sudo ip6tables -A integration_only -j REJECT
-sudo ip6tables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+sudo ip6tables -I OUTPUT 1 -m cgroup --path "$egress_cgroup" -j integration_only
 guard6_installed=true
 
 if curl --noproxy '*' --connect-timeout 2 -s http://198.51.100.1 >/dev/null 2>&1; then
@@ -137,8 +160,12 @@ start_proxy() {
   else
     cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")
   fi
+  local -a database_env=("DATABASE_URL=${INTEGRATION_PROXY_DATABASE_URL:-$DATABASE_URL}")
+  if [ -n "$INTEGRATION_PROXY_READ_REPLICA_URL" ]; then
+    database_env+=("DATABASE_URL_READ_REPLICA=$INTEGRATION_PROXY_READ_REPLICA_URL")
+  fi
   setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
-    DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+    "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
@@ -185,6 +212,15 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
+node_files=()
+if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
+  split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
+    | circleci tests split --split-by=timings --timings-type=filename)"
+  read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
+  test "${#node_files[@]}" -gt 0
+  printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
+fi
+
 env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
   DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
@@ -195,7 +231,10 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
-  .venv/bin/python tests/integration/run.py "$suite" --results "$results"
+  INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
+  INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
+  INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
 
 if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
   for covered_pid in "$proxy_pid" "$peer_pid"; do

.circleci/scripts/unit_selection.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+flag="${1:?usage: unit_selection.sh <codecov flag>}"
+
+legacy_flags=(
+  caching-local
+  core-utils
+  enterprise-package
+  enterprise-routing
+  integrations
+  llm-other-providers
+  llm-vertex-ai
+  mcp-integration
+  misc
+  proxy-db-auth-checks
+  proxy-db-budgets
+  proxy-db-custom-logging
+  proxy-db-db-and-spend
+  proxy-db-endpoints-and-responses
+  proxy-db-guardrails-hooks
+  proxy-db-jwt-and-keys
+  proxy-db-key-generation
+  proxy-db-logging-misc
+  proxy-db-proxy-runtime
+  proxy-db-proxy-server-core
+  proxy-db-proxy-utils
+  proxy-extras
+  proxy-infra
+  responses-caching-types
+)
+
+legacy_paths() {
+  case "$1" in
+    caching-local) echo tests/unit/caching ;;
+    core-utils) echo tests/unit/litellm_core_utils ;;
+    enterprise-package)
+      echo tests/unit/enterprise/integrations
+      echo tests/unit/enterprise/proxy/auth
+      echo tests/unit/enterprise/proxy/guardrails
+      echo tests/unit/enterprise/proxy/hooks
+      echo tests/unit/enterprise/proxy/management_endpoints
+      echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
+      echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
+    enterprise-routing)
+      echo tests/unit/google_genai
+      echo tests/unit/router_strategy
+      echo tests/unit/router_utils
+      echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
+      echo tests/unit/enterprise/enterprise_callbacks/send_emails
+      echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
+      echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+      echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
+      echo tests/unit/enterprise/proxy/test_enterprise_routes.py
+      echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
+      echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
+      echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
+    integrations) echo tests/unit/integrations ;;
+    llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
+    llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
+    mcp-integration)
+      echo tests/unit/experimental_mcp_client
+      echo tests/unit/proxy/_experimental/mcp_server
+      echo tests/unit/responses/mcp
+      echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
+    misc)
+      find tests/unit -maxdepth 1 -name 'test_*.py'
+      echo tests/unit/test_router
+      echo tests/unit/a2a_protocol
+      echo tests/unit/batches
+      echo tests/unit/chat_completions
+      echo tests/unit/completion_extras
+      echo tests/unit/containers
+      echo tests/unit/embeddings
+      echo tests/unit/endpoints
+      echo tests/unit/files
+      echo tests/unit/images
+      echo tests/unit/interactions
+      echo tests/unit/messages
+      echo tests/unit/rag
+      echo tests/unit/rerank_api
+      echo tests/unit/rust_bridge
+      echo tests/unit/secret_managers
+      echo tests/unit/vector_stores
+      echo tests/unit/videos ;;
+    proxy-db-auth-checks)
+      echo tests/unit/proxy/auth/test_auth_checks.py
+      echo tests/unit/proxy/auth/test_user_api_key_auth.py
+      echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
+    proxy-db-budgets)
+      echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+      echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
+      echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
+    proxy-db-custom-logging)
+      echo tests/unit/proxy/test_custom_callback_input.py
+      echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
+    proxy-db-db-and-spend)
+      echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
+      echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
+      echo tests/unit/proxy/db/test_update_daily_tag_spend.py
+      echo tests/unit/proxy/test_db_schema_changes.py
+      echo tests/unit/proxy/test_prisma_client_backoff_retry.py
+      echo tests/unit/proxy/test_update_spend.py
+      echo tests/unit/skills/test_skills_db.py ;;
+    proxy-db-endpoints-and-responses)
+      echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
+      echo tests/unit/proxy/common_utils/test_check_batch_cost.py
+      echo tests/unit/proxy/common_utils/test_check_responses_cost.py
+      echo tests/unit/proxy/common_utils/test_realtime_cache.py
+      echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
+      echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
+      echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
+      echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
+      echo tests/unit/proxy/response_polling/test_response_polling_handler.py
+      echo tests/unit/proxy/test_custom_tokenizer_bug.py
+      echo tests/unit/proxy/test_get_favicon.py
+      echo tests/unit/proxy/test_get_image.py
+      echo tests/unit/proxy/test_prompt_test_endpoint.py
+      echo tests/unit/proxy/test_reducto_ocr_route.py
+      echo tests/unit/proxy/test_response_polling_pre_call_checks.py
+      echo tests/unit/proxy/test_ui_path_detection.py ;;
+    proxy-db-guardrails-hooks)
+      echo tests/unit/proxy/hooks/test_banned_keyword_list.py
+      echo tests/unit/proxy/test_proxy_setting_guardrails.py
+      echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
+    proxy-db-jwt-and-keys)
+      echo tests/unit/proxy/auth/test_jwt.py
+      echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
+      echo tests/unit/proxy/test_proxy_custom_auth.py ;;
+    proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
+    proxy-db-logging-misc)
+      echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
+      echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
+      echo tests/unit/proxy/test_proxy_reject_logging.py ;;
+    proxy-db-proxy-runtime)
+      echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
+      echo tests/unit/proxy/auth/test_proxy_routes.py
+      echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
+      echo tests/unit/proxy/test_proxy_config_unit_test.py
+      echo tests/unit/proxy/test_proxy_token_counter.py
+      echo tests/unit/proxy/test_server_root_path.py ;;
+    proxy-db-proxy-server-core)
+      echo tests/unit/proxy/test_aproxy_startup.py
+      echo tests/unit/proxy/test_proxy_server.py ;;
+    proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
+    proxy-extras) echo tests/unit/litellm_proxy_extras ;;
+    proxy-infra) echo tests/unit/gateway ;;
+    responses-caching-types)
+      find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
+      echo tests/unit/types ;;
+    *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
+  esac
+}
+
+expand() {
+  while read -r path; do
+    if [ -d "$path" ]; then
+      find "$path" -name 'test_*.py'
+    elif [ -f "$path" ]; then
+      echo "$path"
+    else
+      echo "unit_selection.sh: $path does not exist" >&2
+      exit 1
+    fi
+  done
+}
+
+if [ "$flag" = unit ]; then
+  comm -23 \
+    <(find tests/unit -name 'test_*.py' | sort) \
+    <(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
+  exit 0
+fi
+
+legacy_paths "$flag" | expand | sort

.circleci/tests.yml

@@ -74,6 +74,7 @@ commands:
     steps:
       - run:
           name: Install Codecov CLI (pinned v11.3.1)
+          when: always
           command: |
             curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
             curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
@@ -90,7 +91,6 @@ commands:
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
   setup_test_deps:
     steps:
-      - checkout
       - install_uv
       - install_rust
       - restore_cache:
@@ -165,42 +165,72 @@ commands:
 jobs:
   unit:
     parameters:
-      tests_path:
-        type: string
-        default: tests/unit
       flag:
         type: string
         default: unit
       shards:
         type: integer
         default: 6
+      workers:
+        type: integer
+        default: 4
+      dist:
+        type: string
+        default: loadscope
       base_ref:
         type: string
         default: ""
       pull_request_url:
         type: string
         default: ""
+      legacy_mcp_peer:
+        type: boolean
+        default: false
+      reruns:
+        type: integer
+        default: 0
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
     parallelism: << parameters.shards >>
     environment:
+      COVERAGE_CORE: sysmon
       LITELLM_LOCAL_MODEL_COST_MAP: "True"
     steps:
-      - setup_test_deps
+      - checkout
       - skip_unless_relevant:
           base_ref: << parameters.base_ref >>
           pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
+      - when:
+          condition: << parameters.legacy_mcp_peer >>
+          steps:
+            - run:
+                name: Install MCP SDK1 peer
+                command: |
+                  uv venv --python 3.12 .venv-mcp-peer
+                  uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+                  echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
       - run:
-          name: "Run << parameters.tests_path >> shard"
+          name: "Run << parameters.flag >> shard"
           no_output_timeout: 20m
           command: |
             mkdir -p test-results/<< parameters.flag >>
-            mapfile -t files < <(find << parameters.tests_path >> -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)
-            if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.tests_path >> files; nothing to run"; exit 0; fi
+            selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
+            [ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
+            shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
+            [ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
+            mapfile -t files < <(printf '%s\n' "${shard}")
+            xdist_args=()
+            if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
+            rerun_args=(-p no:rerunfailures)
+            if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
+            test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
+            if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
             set +e
-            uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
+            env -i "${test_env[@]}" \
+              uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
@@ -224,6 +254,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
+      - checkout
       - setup_test_deps
       - run:
           name: Checkout litellm-docs
@@ -250,16 +281,17 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - setup_test_deps
+      - checkout
       - skip_unless_relevant:
           base_ref: << parameters.base_ref >>
           pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
       - start_redis
       - run:
           name: Run owned integration contracts
-          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
           no_output_timeout: 15m
       - run:
           name: Stop owned database and Redis
@@ -282,6 +314,98 @@ workflows:
       - unit:
           base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
           pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-<< matrix.flag >>
+          shards: 1
+          workers: 2
+          reruns: 2
+          matrix:
+            parameters:
+              flag: [caching-local, proxy-extras, enterprise-routing]
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-mcp-integration
+          flag: mcp-integration
+          shards: 1
+          workers: 2
+          legacy_mcp_peer: true
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-<< matrix.flag >>
+          shards: 1
+          reruns: 2
+          matrix:
+            parameters:
+              flag:
+                - enterprise-package
+                - proxy-infra
+                - responses-caching-types
+                - proxy-db-auth-checks
+                - proxy-db-jwt-and-keys
+                - proxy-db-proxy-server-core
+                - proxy-db-proxy-runtime
+                - proxy-db-custom-logging
+                - proxy-db-logging-misc
+                - proxy-db-db-and-spend
+                - proxy-db-guardrails-hooks
+                - proxy-db-budgets
+                - proxy-db-endpoints-and-responses
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-llm-vertex-ai
+          flag: llm-vertex-ai
+          shards: 2
+          workers: 1
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-llm-other-providers
+          flag: llm-other-providers
+          shards: 3
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-core-utils
+          flag: core-utils
+          shards: 2
+          reruns: 1
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-integrations
+          flag: integrations
+          shards: 2
+          reruns: 3
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-misc
+          flag: misc
+          shards: 2
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-proxy-db-proxy-utils
+          flag: proxy-db-proxy-utils
+          shards: 1
+          reruns: 2
+          dist: worksteal
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-proxy-db-key-generation
+          flag: proxy-db-key-generation
+          shards: 1
+          workers: 0
+          reruns: 2
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
       - documentation
       - integration:
           name: integration-<< matrix.suite >>

.github/merge-smoke-tests.json

@@ -1,15 +1,15 @@
 {
   "cases": {
-    "CHAT-JSON": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
-    "CHAT-TEXT-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
-    "CHAT-TOOL-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
+    "CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
+    "CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
+    "CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
     "MODEL-ALLOW": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_allows_listed_model_for_key",
     "MODEL-DENY": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
-    "COST-EXPLICIT": "tests/test_litellm/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
-    "COST-ZERO": "tests/test_litellm/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
-    "LOG-CONTENT-ON": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
-    "LOG-CONTENT-OFF": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
-    "CALLBACK-SUCCESS": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
-    "CALLBACK-FAILURE": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
+    "COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
+    "COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
+    "LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
+    "LOG-CONTENT-OFF": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
+    "CALLBACK-SUCCESS": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
+    "CALLBACK-FAILURE": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
   }
 }

.github/pull_request_template.md

@@ -6,7 +6,8 @@
 
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
+     If the PR intentionally changes what existing users see or how a screen behaves, add a line under the bullets that starts "Intentional product change:" describing what changes, why, and what users lose. Reviewers must never have to infer a deliberate UX change from the diff -->
 
 Problem this solves:
 
@@ -28,7 +29,8 @@ How it solves it:
      No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
      Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
      If the bug had a security or authorization consequence, end each list with what another user could or could no longer do
-     Regenerate this section whenever new commits change the PR's behavior, so it never describes an older revision
+     Regenerate this section, screenshots included, whenever new commits change the PR's behavior, so it never describes an older revision
+     If the PR changes what an Admin UI page shows, embed a before and an after screenshot of that page right after its list, taken at the same URL on the same data, with the rows, fields, or controls that changed boxed in red so a reader spots the difference without reading the steps. These are the UI screenshots for Screenshots / Proof of Fix too: embed them once here and have that section's Before and After steps point back to them instead of repeating the images
 
 Example:
 
@@ -52,7 +54,7 @@ After: the same request comes back with real token counts, so the dashboard show
 
 ## Affected release
 
-<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Drop the section otherwise -->
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1". Add the `backport-stable` label only when the regression is a P0, meaning its Linear ticket is Urgent (a security hole however narrow, data loss, or a crash or outage for every user on that version), because every labeled PR must be cherry-picked onto the baking rc line before the stable can be tagged; every other regression fix ships in the next rc unlabeled. Drop the section otherwise -->
 
 ## Linear ticket
 
@@ -63,7 +65,7 @@ After: the same request comes back with real token counts, so the dashboard show
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
 - [ ] I have added meaningful tests
-- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
+- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
 - [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
 - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)

.github/scripts/assert_ci_coverage.py

@@ -34,7 +34,6 @@ GLOB_CHARS = frozenset("*?")
 # tests has to be named by some shard or it runs nowhere. A child listed here is
 # itself decomposed one level deeper and is checked through its own entry.
 SHARDED_ROOTS: tuple[str, ...] = (
-    "tests/proxy_unit_tests",
     "tests/test_litellm",
     "tests/test_litellm/proxy",
 )
@@ -120,6 +119,13 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
     )
 
 
+def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
+    script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
+    if not script.is_file():
+        return frozenset()
+    return frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(_uncommented(script.read_text())))
+
+
 def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
     return frozenset(
         match.group(0)
@@ -510,7 +516,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         str(path.relative_to(repo_root))
         for folders in groups.values()
         for folder in folders
-        for path in (integration_root / folder).glob("test_*.py")
+        for path in (integration_root / folder).rglob("test_*.py")
     )
     browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
     browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()
@@ -611,7 +617,10 @@ def main() -> int:
     scalars = _all_scalars()
 
     integration_paths, ownership_findings = _integration_ownership()
-    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings
+    test_findings = (
+        _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
+        + ownership_findings
+    )
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
 

.github/scripts/read_rc_version.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Print `version=X.Y.0` from [project].version in pyproject.toml for $GITHUB_OUTPUT.
+
+Usage
+-----
+    python3 read_rc_version.py [path/to/pyproject.toml] >> "$GITHUB_OUTPUT"
+
+Exit code 1 with a `::error::` line on stderr when the version is not an X.Y.0 release.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+from typing import Final
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+RELEASE_VERSION: Final = re.compile(r"[0-9]+\.[0-9]+\.0")
+
+
+def read_version(pyproject: pathlib.Path) -> str:
+    with pyproject.open("rb") as f:
+        return tomllib.load(f)["project"]["version"]
+
+
+def main(argv: list[str]) -> int:
+    pyproject: Final = pathlib.Path(argv[1]) if len(argv) > 1 else pathlib.Path("pyproject.toml")
+    version: Final = read_version(pyproject)
+    if RELEASE_VERSION.fullmatch(version) is None:
+        print(  # noqa: T201  # the ::error:: line to stderr is the workflow's failure signal
+            f"::error::pyproject.toml version {version} is not an X.Y.0 release version", file=sys.stderr
+        )
+        return 1
+    print(f"version={version}")  # noqa: T201  # stdout line is appended to $GITHUB_OUTPUT
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/workflows/_test-unit-base.yml

@@ -13,6 +13,16 @@ on:
           have its path existence-checked like any other token.
         required: true
         type: string
+      unit-flag:
+        description: >-
+          Codecov flag of the `.circleci/tests.yml` job that now owns part of
+          this shard. The shard also runs the files
+          `.circleci/scripts/unit_selection.sh` lists for the flag, on every
+          event, because the CircleCI pipeline is manual-only while the tests
+          migrate.
+        required: false
+        type: string
+        default: ""
       workers:
         description: "Number of pytest-xdist workers"
         required: false
@@ -92,6 +102,7 @@ jobs:
       pull-requests: read
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
+      has-coverage: ${{ steps.tests.outputs.has-coverage }}
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -160,10 +171,12 @@ jobs:
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
+        id: tests
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
+          UNIT_FLAG: ${{ inputs.unit-flag }}
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
@@ -171,9 +184,18 @@ jobs:
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
+          echo "has-coverage=false" >> "$GITHUB_OUTPUT"
+          selection="${TEST_PATH}"
+          if [ -n "${UNIT_FLAG}" ]; then
+            selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')"
+          fi
+          if [ -z "${selection// /}" ]; then
+            echo "shard selection is empty; nothing to run"
+            exit 0
+          fi
           pytest_args=()
           existing_paths=0
-          for token in ${TEST_PATH:?}; do
+          for token in ${selection}; do
             case "${token}" in
               -*) pytest_args+=("${token}") ;;
               *)
@@ -187,7 +209,7 @@ jobs:
             esac
           done
           if [ "${existing_paths}" -eq 0 ]; then
-            echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
+            echo "No path in the selection exists (${selection}); nothing to run"
             exit 0
           fi
           xdist_args=()
@@ -209,8 +231,11 @@ jobs:
             --cov-config=pyproject.toml
           status=$?
           set -e
+          if [ -f coverage.xml ]; then
+            echo "has-coverage=true" >> "$GITHUB_OUTPUT"
+          fi
           if [ "$status" -eq 5 ]; then
-            echo "pytest collected no tests from ${TEST_PATH}; passing"
+            echo "pytest collected no tests from ${selection}; passing"
             exit 0
           fi
           exit "$status"
@@ -226,7 +251,7 @@ jobs:
   upload-coverage:
     name: Upload coverage to Codecov
     needs: run
-    if: always() && needs.run.outputs.decision != 'skip'
+    if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
     runs-on: ubuntu-latest
     permissions:
       contents: read

.github/workflows/compat-matrix-image.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - tests/e2e/claude_code/cron_vm/**
+      - tests/e2e/claude_code/pr_gate_version_resolver.py
       - .github/workflows/compat-matrix-image.yml
   workflow_dispatch:
 
@@ -28,6 +29,14 @@ jobs:
       - name: Build the Render cron image
         run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
 
-      - name: Run the pinned binaries as the cron user
+      - name: Resolve and install the Claude Code CLI as the cron user
         run: |
-          docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version'
+          docker run --rm compat-matrix:${{ github.sha }} bash -c '
+            set -euo pipefail
+            whoami
+            gh --version
+            uv --version
+            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
+            /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
+            /tmp/claude-cli/claude --version
+          '

.github/workflows/create-rc-branch.yml

@@ -0,0 +1,66 @@
+name: Create RC Branch
+
+on:
+  schedule:
+    - cron: "0 3 * * 5"
+      timezone: "America/Los_Angeles"
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  create-rc-branch:
+    name: Create RC Branch
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Require main
+        env:
+          REF: ${{ github.ref }}
+        run: |
+          if [ "$REF" != "refs/heads/main" ]; then
+            echo "::error::rc branches are cut from refs/heads/main only, got $REF"
+            exit 1
+          fi
+
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Read release version
+        id: version
+        run: python3 .github/scripts/read_rc_version.py >> "$GITHUB_OUTPUT"
+
+      - name: Create rc branch
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const branchName = `rc/${process.env.VERSION}`;
+            const ref = `heads/${branchName}`;
+
+            const existing = await github.rest.git.getRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref,
+            }).catch((error) => {
+              if (error.status === 404) {
+                return null;
+              }
+              throw error;
+            });
+            if (existing !== null) {
+              core.setFailed(`Branch ${branchName} already exists at ${existing.data.object.sha}; leaving it untouched`);
+              return;
+            }
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/${ref}`,
+              sha: context.sha,
+            });
+            core.info(`Created branch ${branchName} at ${context.sha}`);

.github/workflows/test-code-quality.yml

@@ -146,6 +146,9 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_unbounded_in_lists (fails on findings not in the baseline)
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -180,6 +180,18 @@ jobs:
             echo "No changed tests/e2e Python files; skipping."
           fi
 
+      - name: Run the claude_code harness unit tests
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/claude_code/**/*.py' ':(glob)tests/e2e/*.py' tests/e2e/claude_code/cron_vm/install_claude_code.sh pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
+            echo "No changed claude_code harness files; skipping."
+            exit 0
+          fi
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
+          tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
+          PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures tests/e2e/claude_code/_*_unit_tests
+
       - name: Check for circular imports
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-redis-compat.yml

@@ -8,9 +8,13 @@ on:
     paths:
       - "litellm/_redis.py"
       - "litellm/_redis_credential_provider.py"
-      - "tests/test_litellm/test_redis.py"
+      - "litellm/caching/redis_cache.py"
+      - "litellm/caching/evicted_client_closer.py"
+      - "tests/unit/test_redis.py"
       - "tests/local_testing/test_caching.py"
-      - "tests/test_litellm/caching/test_redis_connection_pool.py"
+      - "tests/unit/caching/test_redis_connection_pool.py"
+      - "tests/unit/caching/test_redis_cluster_cache.py"
+      - "tests/unit/caching/test_evicted_client_closer.py"
       - ".github/workflows/test-redis-compat.yml"
       - "pyproject.toml"
       - "uv.lock"
@@ -80,8 +84,10 @@ jobs:
         run: |
           redis-server --version
           uv run --no-sync pytest \
-            tests/test_litellm/test_redis.py \
-            tests/test_litellm/caching/test_redis_connection_pool.py \
+            tests/unit/test_redis.py \
+            tests/unit/caching/test_redis_connection_pool.py \
+            tests/unit/caching/test_redis_cluster_cache.py \
+            tests/unit/caching/test_evicted_client_closer.py \
             tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
             tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
             --tb=short -vv \

.github/workflows/test-rust.yml

@@ -14,7 +14,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -24,7 +23,7 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
   pull_request:
     branches:
@@ -42,7 +41,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/ocr/**"
       - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
@@ -52,7 +50,7 @@ on:
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
       - ".github/scripts/verify_linux_native_wheel.py"
-      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
+      - "tests/unit/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
 
 permissions:
@@ -171,7 +169,7 @@ jobs:
         env:
           RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
-      - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+      - run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
 
       - name: Run pytest tests/test_litellm_rust with the compiled extension
         run: make test-rust-extension

.github/workflows/test-unit-proxy-db.yml

@@ -20,6 +20,13 @@ concurrency:
 # rather than alphabetical letter ranges. Adding a new test file means adding it
 # to whichever group it belongs to, not reshuffling slices.
 #
+# `.circleci/tests.yml` runs each group's files on same-repo events under the
+# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
+# the file lists. That pipeline is manual-only while the tests migrate, so
+# `unit-flag` makes the shard run that list on every event. `test-path` keeps
+# the files that still reach real providers and never left
+# tests/proxy_unit_tests.
+#
 # Design targets:
 #   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
@@ -58,7 +65,7 @@ jobs:
   proxy-db:
     needs: assert-shard-coverage
     # Display only the semantic shard name in the checks UI instead of GHA's
-    # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
+    # default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
     # which includes every matrix field and gets truncated past the test-path.
     name: ${{ matrix.test-group }}
     permissions:
@@ -71,132 +78,93 @@ jobs:
         include:
           # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
-            test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
+            test-path: ""
+            unit-flag: proxy-db-key-generation
             workers: 0
             dist: loadscope
             timeout: 20
 
           # ---- auth: split into 2 shards ----
           - test-group: auth-checks
-            test-path: >-
-              tests/proxy_unit_tests/test_auth_checks.py
-              tests/proxy_unit_tests/test_user_api_key_auth.py
-              tests/proxy_unit_tests/test_deprecated_key_grace_period.py
+            test-path: ""
+            unit-flag: proxy-db-auth-checks
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: jwt-and-keys
-            test-path: >-
-              tests/proxy_unit_tests/test_jwt.py
-              tests/proxy_unit_tests/test_jwt_key_mapping.py
-              tests/proxy_unit_tests/test_proxy_custom_auth.py
-              tests/proxy_unit_tests/test_key_generate_dynamodb.py
+            test-path: ""
+            unit-flag: proxy-db-jwt-and-keys
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- test_proxy_utils.py, single shard, worksteal distribution ----
           - test-group: proxy-utils
-            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            test-path: ""
+            unit-flag: proxy-db-proxy-utils
             workers: 4
             dist: worksteal
             timeout: 15
 
           # ---- proxy server: split into 2 shards ----
           - test-group: proxy-server-core
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_server.py
-              tests/proxy_unit_tests/test_aproxy_startup.py
+            test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
+            unit-flag: proxy-db-proxy-server-core
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: proxy-runtime
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_config_unit_test.py
-              tests/proxy_unit_tests/test_proxy_routes.py
-              tests/proxy_unit_tests/test_server_root_path.py
-              tests/proxy_unit_tests/test_proxy_pass_user_config.py
-              tests/proxy_unit_tests/test_proxy_token_counter.py
-              tests/proxy_unit_tests/test_request_size_limit_middleware.py
-              tests/proxy_unit_tests/test_multipart_bypass_repro.py
+            test-path: ""
+            unit-flag: proxy-db-proxy-runtime
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- logging: split into 2 shards ----
           - test-group: custom-logging
-            test-path: >-
-              tests/proxy_unit_tests/test_custom_callback_input.py
-              tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
-              tests/proxy_unit_tests/test_proxy_custom_logger.py
+            test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
+            unit-flag: proxy-db-custom-logging
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: logging-misc
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_reject_logging.py
-              tests/proxy_unit_tests/test_audit_logs_proxy.py
-              tests/proxy_unit_tests/test_search_api_logging.py
+            test-path: ""
+            unit-flag: proxy-db-logging-misc
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: db-and-spend
-            test-path: >-
-              tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
-              tests/proxy_unit_tests/test_db_schema_changes.py
-              tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
-              tests/proxy_unit_tests/test_skills_db.py
-              tests/proxy_unit_tests/test_update_daily_tag_spend.py
-              tests/proxy_unit_tests/test_update_spend.py
-              tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
+            test-path: ""
+            unit-flag: proxy-db-db-and-spend
             workers: 4
             dist: loadscope
             timeout: 15
 
           # ---- guardrails + budget + hooks: split into 2 ----
           - test-group: guardrails-hooks
-            test-path: >-
-              tests/proxy_unit_tests/test_proxy_setting_guardrails.py
-              tests/proxy_unit_tests/test_banned_keyword_list.py
-              tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
+            test-path: ""
+            unit-flag: proxy-db-guardrails-hooks
             workers: 4
             dist: loadscope
             timeout: 15
           - test-group: budgets
-            test-path: >-
-              tests/proxy_unit_tests/test_default_end_user_budget_simple.py
-              tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
-              tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
+            test-path: ""
+            unit-flag: proxy-db-budgets
             workers: 4
             dist: loadscope
             timeout: 15
 
           - test-group: endpoints-and-responses
-            test-path: >-
-              tests/proxy_unit_tests/test_blog_posts_endpoint.py
-              tests/proxy_unit_tests/test_models_fallback_endpoint.py
-              tests/proxy_unit_tests/test_google_endpoint_routing.py
-              tests/proxy_unit_tests/test_google_gemini_proxy_request.py
-              tests/proxy_unit_tests/test_gemini_agents_endpoints.py
-              tests/proxy_unit_tests/test_get_favicon.py
-              tests/proxy_unit_tests/test_get_image.py
-              tests/proxy_unit_tests/test_reducto_ocr_route.py
-              tests/proxy_unit_tests/test_ui_path_detection.py
-              tests/proxy_unit_tests/test_prompt_test_endpoint.py
-              tests/proxy_unit_tests/test_check_batch_cost.py
-              tests/proxy_unit_tests/test_check_responses_cost.py
-              tests/proxy_unit_tests/test_response_polling_handler.py
-              tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
-              tests/proxy_unit_tests/test_realtime_cache.py
-              tests/proxy_unit_tests/test_proxy_exception_mapping.py
-              tests/proxy_unit_tests/test_custom_tokenizer_bug.py
+            test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
+            unit-flag: proxy-db-endpoints-and-responses
             workers: 4
             dist: loadscope
             timeout: 15
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}
+      unit-flag: ${{ matrix.unit-flag }}
       workers: ${{ matrix.workers }}
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}

.github/workflows/test-unit.yml

@@ -31,10 +31,14 @@ concurrency:
 # number, so a partially-specified entry would fail the call rather than fall
 # back to the default.
 #
-# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
-# already a matrix and carries a shard-coverage guard that reads that file by
-# name. Folding it in here is a follow-up, together with generalising that guard
-# into assert_ci_coverage.py.
+# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already
+# a matrix and carries a shard-coverage guard that reads that file by name.
+# Folding it in here is a follow-up, together with generalising that guard into
+# assert_ci_coverage.py.
+#
+# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the
+# shard under the same Codecov flag. That pipeline is manual-only while the
+# tests migrate, so the shard also runs those files on every event.
 jobs:
   unit:
     name: ${{ matrix.shard }}
@@ -48,7 +52,8 @@ jobs:
         include:
           - shard: mcp-integration
             artifact-name: mcp-integration
-            test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client"
+            test-path: "tests/mcp_tests"
+            unit-flag: mcp-integration
             workers: 2
             reruns: 0
             timeout-minutes: 20
@@ -56,7 +61,8 @@ jobs:
 
           - shard: core-utils
             artifact-name: core-utils
-            test-path: "tests/test_litellm/litellm_core_utils"
+            test-path: ""
+            unit-flag: core-utils
             workers: 2
             reruns: 1
             timeout-minutes: 20
@@ -64,11 +70,8 @@ jobs:
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
-            test-path: >-
-              tests/test_litellm/enterprise
-              tests/test_litellm/google_genai
-              tests/test_litellm/router_utils
-              tests/test_litellm/router_strategy
+            test-path: ""
+            unit-flag: enterprise-routing
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -76,7 +79,8 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: "tests/test_litellm/integrations"
+            test-path: ""
+            unit-flag: integrations
             workers: 2
             reruns: 3
             timeout-minutes: 20
@@ -84,7 +88,8 @@ jobs:
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
-            test-path: "tests/test_litellm/llms/vertex_ai"
+            test-path: ""
+            unit-flag: llm-vertex-ai
             workers: 1
             reruns: 2
             timeout-minutes: 20
@@ -92,7 +97,8 @@ jobs:
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
-            test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            test-path: ""
+            unit-flag: llm-other-providers
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -101,25 +107,8 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/test_litellm/batches
-              tests/test_litellm/secret_managers
-              tests/test_litellm/a2a_protocol
-              tests/test_litellm/chat_completions
-              tests/test_litellm/completion_extras
-              tests/test_litellm/containers
-              tests/test_litellm/endpoints
-              tests/test_litellm/images
-              tests/test_litellm/interactions
-              tests/test_litellm/messages
-              tests/test_litellm/embeddings
-              tests/test_litellm/ocr
-              tests/test_litellm/passthrough
-              tests/test_litellm/rag
-              tests/test_litellm/rerank_api
-              tests/test_litellm/rust_bridge
-              tests/test_litellm/vector_stores
-              tests/test_litellm/videos
               tests/test_litellm/test_*.py
+            unit-flag: misc
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -199,7 +188,7 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
-              tests/test_gateway
+            unit-flag: proxy-infra
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -207,11 +196,8 @@ jobs:
 
           - shard: caching-local
             artifact-name: caching-local
-            test-path: >-
-              tests/local_testing/test_cache_preset_key.py
-              tests/local_testing/test_caching_handler.py
-              tests/local_testing/test_responses_stream_cache_keys.py
-              tests/local_testing/test_unit_test_caching.py
+            test-path: ""
+            unit-flag: caching-local
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -219,7 +205,8 @@ jobs:
 
           - shard: proxy-extras
             artifact-name: proxy-extras
-            test-path: "tests/litellm-proxy-extras"
+            test-path: ""
+            unit-flag: proxy-extras
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -227,7 +214,8 @@ jobs:
 
           - shard: enterprise-package
             artifact-name: enterprise-package
-            test-path: "tests/enterprise"
+            test-path: ""
+            unit-flag: enterprise-package
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -235,10 +223,8 @@ jobs:
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
-            test-path: >-
-              tests/test_litellm/responses
-              tests/test_litellm/caching
-              tests/test_litellm/types
+            test-path: ""
+            unit-flag: responses-caching-types
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -246,6 +232,7 @@ jobs:
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}
+      unit-flag: ${{ matrix.unit-flag || '' }}
       workers: ${{ matrix.workers }}
       reruns: ${{ matrix.reruns }}
       timeout-minutes: ${{ matrix.timeout-minutes }}

AGENTS.md

@@ -27,7 +27,7 @@ Never test structure of code only function of it
 
 A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
 
-`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
+`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
 
@@ -96,6 +96,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
   - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
   - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
+  - Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

ARCHITECTURE.md

@@ -255,7 +255,7 @@ Conventions to follow when touching this layer:
 | Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. |
 | Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. |
 
-To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`.
+To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/unit/repositories/`.
 
 ---
 
@@ -336,7 +336,7 @@ Each translation is isolated in its own file, making it easy to test and modify
 | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
 | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
 | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
-| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
 | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
 | `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
 | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |

CONTRIBUTING.md

@@ -14,7 +14,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
+  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
 
 #### UI PRs
 
@@ -72,7 +72,7 @@ make format
 make lint
 
 # Run the tests covering your change (CI runs the full suite)
-uv run pytest tests/test_litellm/<your_test_file>.py -v
+uv run pytest tests/unit/<your_test_file>.py -v
 
 # Commit your changes (must follow Conventional Commits — see above)
 git add .
@@ -88,7 +88,7 @@ git push origin feature/your-feature
 
 ### Where to Add Tests
 
-Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+Add your tests to the [`tests/unit/` directory](https://github.com/BerriAI/litellm/tree/main/tests/unit).
 
 - This directory mirrors the structure of the `litellm/` directory
 - **Only add mocked tests** - no real LLM API calls in this directory
@@ -96,10 +96,10 @@ Add your tests to the [`tests/test_litellm/` directory](https://github.com/Berri
 
 ### File Naming Convention
 
-The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+The `tests/unit/` directory follows the same structure as `litellm/`:
 
 - `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
-- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+- `litellm/utils.py` → `tests/unit/test_utils.py`
 
 ### Example Test
 
@@ -125,10 +125,10 @@ def test_your_feature():
 
 Run the tests covering your change:
 ```bash
-uv run pytest tests/test_litellm/test_your_file.py -v
+uv run pytest tests/unit/test_your_file.py -v
 ```
 
-`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
+`tests/unit` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
 
 If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
 

Makefile

@@ -42,7 +42,7 @@ help:
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit          - Run unit tests (tests/unit and tests/test_litellm)"
 	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
 	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
 	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
@@ -51,8 +51,8 @@ help:
 	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
 	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
 	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
-	@echo "  make test-proxy-unit-a  - Run proxy_unit_tests (a-o, ~20 files)"
-	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
+	@echo "  make test-proxy-unit-a  - Run tests/unit/proxy (a-o)"
+	@echo "  make test-proxy-unit-b  - Run tests/unit/proxy (p-z)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
@@ -301,7 +301,7 @@ test-rust-extension:
 	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
 	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
 	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
-		--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
+		--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
 		litellm.rust_bridge._native && \
 	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
 	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
@@ -310,11 +310,11 @@ test: install-test-deps
 	$(UV_RUN) pytest tests/
 
 test-unit: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
+	$(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4
 
 # Matrix test targets (matching CI workflow groups)
 test-unit-llms: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/llms --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-guardrails: install-test-deps
 	$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
@@ -326,23 +326,23 @@ test-unit-proxy-misc: install-test-deps
 	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20
 
 test-unit-core-utils: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20
 
 test-unit-other: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
 
-# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+# Proxy unit tests (tests/unit/proxy split alphabetically)
 test-proxy-unit-a: install-test-deps
-	$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy --ignore-glob='tests/unit/proxy/test_[p-z]*.py' --tb=short -vv -n 2 --durations=20
 
 test-proxy-unit-b: install-test-deps
-	$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py tests/unit/skills --tb=short -vv -n 2 --durations=20
 
 test-integration: install-test-deps
 	$(UV_RUN) pytest tests/ -k "not test_litellm"

README.md

@@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json

@@ -1697,6 +1697,63 @@
       "title": "litellm_video_duration_seconds_metric rate",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "Share of the provider's bill LiteLLM captured as spend over the scheduled capture-rate check's window (needs general_settings.spend_capture_rate_check); NaN while no rate is available",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 107
+      },
+      "id": 111,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "max by (api_provider) (litellm_spend_capture_rate)",
+          "legendFormat": "{{api_provider}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "litellm_spend_capture_rate",
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
       "gridPos": {

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md

@@ -1,8 +1,8 @@
 # LiteLLM All Prometheus Metrics dashboard
 
-Every `litellm_*` metric family the proxy can expose on `/metrics` (134 families across 95 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
+Every `litellm_*` metric family the proxy can expose on `/metrics` (136 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
 
-Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
 
 The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
 

cookbook/litellm_proxy_server/mcp/README.md

@@ -0,0 +1,37 @@
+# Publish MCP servers in the AI Hub
+
+Set `litellm_settings.public_mcp_servers` to the concrete IDs of the servers you want listed in the public AI Hub. Pin `server_id` in each configuration entry so the publication list stays stable across deployments
+
+```yaml
+mcp_servers:
+  documentation:
+    server_id: documentation-mcp
+    url: https://mcp.example.com/mcp
+    transport: http
+    available_on_public_internet: true
+
+litellm_settings:
+  public_mcp_hub_strict_whitelist: true
+  public_mcp_servers:
+    - documentation-mcp
+```
+
+Use `documentation-mcp`, the `server_id`, in the publication list. The configuration key `documentation`, display names, and aliases are not publication IDs. Database-created servers use the ID returned by `/v1/mcp/server`
+
+The dashboard's **AI Hub > MCP Hub > Manage MCP Hub Visibility** dialog edits this same list. Its YAML example includes the selected server IDs. With database-backed configuration (`store_model_in_db: true`), a value declared in YAML is owned by that file: edit the file and reload, or remove that key from YAML to let the dashboard manage it in the database. File-backed deployments can save the list directly to their configuration file
+
+To remove all explicit entries, save an empty selection in the dialog or configure:
+
+```yaml
+litellm_settings:
+  public_mcp_hub_strict_whitelist: true
+  public_mcp_servers: []
+```
+
+## Hub listing and network access
+
+The **Hub listing** column in AI Hub identifies servers that appear in `/public/mcp_hub`. The dashboard derives this status from the current registry and publication settings. Setting `mcp_info.is_public` on a server does not publish it; that response field is derived metadata. `mcp_info.is_public_explicit` identifies registered servers included in the explicit publication list
+
+Gateway cards and server details show **All Networks** when `available_on_public_internet` is enabled or the server is explicitly published in `public_mcp_servers`. They show **Internal Only** when both are false. The per-server flag defaults to `true`; explicit publication overrides a disabled flag for compatibility. Older proxies that omit the metadata needed to determine access show **Unknown**. These labels describe allowed client IPs; authentication and tool permissions still apply
+
+The default `public_mcp_hub_strict_whitelist: true` lists only registered servers in `public_mcp_servers`. Legacy mode (`false`) additionally lists registered servers with `available_on_public_internet: true`. In legacy mode, clearing the explicit publication list leaves these automatically listed servers visible. Enable strict mode when the publication list should fully determine hub visibility

db_scripts/backfill_key_total_spend.sql

@@ -0,0 +1,43 @@
+-- One-shot backfill of LiteLLM_VerificationToken.total_spend (lifetime spend)
+-- for keys created before the column was introduced in LiteLLM v1.103.0.
+--
+-- The column was added with DEFAULT 0 and no backfill, so keys that predate
+-- the upgrade report lifetime spend below their current period spend. New
+-- deployments do not need this script: total_spend is updated at request
+-- time from the moment the release is deployed. Run it only if you want
+-- pre-upgrade keys to show their historical lifetime spend. It sets lifetime
+-- spend to at least the current spend on every key, active and archived,
+-- because current period spend is a valid lower bound on lifetime spend.
+-- For keys with no budget reset that is already the exact lifetime value;
+-- for resetting keys it only recovers the current period. It is idempotent:
+-- it only touches rows where total_spend is below spend, so re-running is a
+-- no-op. It touches no spend logs and runs in seconds.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. A key "resets" when its own budget_duration IS NOT NULL, or when its
+--    budget_id links to a LiteLLM_BudgetTable row whose budget_duration IS
+--    NOT NULL (a linked budget resets the key's spend each period too). For
+--    those keys this script only recovers the current period;
+--    db_scripts/backfill_key_total_spend_from_spend_logs.sql is an optional
+--    follow-up that rebuilds the earlier periods from LiteLLM_SpendLogs.
+--
+-- 3. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend.sql
+
+UPDATE "LiteLLM_VerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+UPDATE "LiteLLM_DeletedVerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

db_scripts/backfill_key_total_spend_from_spend_logs.sql

@@ -0,0 +1,89 @@
+-- Optional follow-up to db_scripts/backfill_key_total_spend.sql. Run that
+-- script first; this one rebuilds earlier budget periods for the keys it
+-- can only partially fix: keys whose spend resets each period, because their own
+-- budget_duration IS NOT NULL or because their budget_id links to a
+-- LiteLLM_BudgetTable row whose budget_duration IS NOT NULL.
+--
+-- For those keys the "spend" column only covers the current period, so
+-- lifetime spend is reconstructed from LiteLLM_SpendLogs. The join matches
+-- l.api_key against both the stored token and its second sha256
+-- (encode(sha256(convert_to(token, 'UTF8')), 'hex')), because spend logs
+-- written by older paths recorded the re-hashed digest instead of the
+-- token. It is idempotent and never lowers a value: every statement only
+-- touches rows where total_spend is below the rebuilt sum, so re-running is
+-- a no-op, and a key whose log history is shorter than its current period
+-- keeps the value backfill_key_total_spend.sql already gave it.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. It requires spend logs to have been enabled, and coverage is bounded
+--    by maximum_spend_logs_retention_period: spend older than the retention
+--    window is already gone and cannot be recovered.
+--
+-- 3. On a large SpendLogs table the join scan is slow, so run it off peak.
+--
+-- 4. Run it while the proxy is idle (or with traffic paused). The proxy
+--    flushes spend logs in batches, so a request that already raised
+--    total_spend but whose log is still queued is missing from the sum, and
+--    the rebuilt value would be short by that in-flight amount.
+--
+-- 5. A custom token can be deleted and recreated, so the archived table can
+--    hold several lifetimes of one token. The update only rewrites archived
+--    rows that reset, and the log sum covers every lifetime of that token.
+--
+-- 6. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend_from_spend_logs.sql
+
+-- Active keys whose spend resets (own budget_duration, or a linked
+-- LiteLLM_BudgetTable row with one). Rebuild from LiteLLM_SpendLogs,
+-- matching api_key against the stored token and its second sha256 digest.
+UPDATE "LiteLLM_VerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM "LiteLLM_VerificationToken" k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    WHERE k2.budget_duration IS NOT NULL
+       OR k2.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       )
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend;
+
+-- Archived tokens are not unique, so collapse them to one row per token
+-- before joining spend logs; the update then hits every resetting archived
+-- row.
+UPDATE "LiteLLM_DeletedVerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM (
+        SELECT DISTINCT token
+        FROM "LiteLLM_DeletedVerificationToken"
+        WHERE budget_duration IS NOT NULL
+           OR budget_id IN (
+               SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+           )
+    ) k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend
+  AND (k.budget_duration IS NOT NULL
+       OR k.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       ));
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -32,6 +32,7 @@ from litellm.integrations.email_templates.key_rotated_email import (
 from litellm.integrations.email_templates.templates import (
     MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
     SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
+    TEAM_MEMBER_MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
     TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
 )
 from litellm.integrations.email_templates.user_invitation_email import (
@@ -48,6 +49,12 @@ from litellm.secret_managers.main import get_secret_bool
 from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
 
 
+def _max_budget_alert_id(user_info: CallInfo) -> str:
+    if user_info.event_group == Litellm_EntityType.TEAM_MEMBER:
+        return f"team_member:{user_info.user_id}:{user_info.team_id}"
+    return user_info.token or user_info.user_id or "default_id"
+
+
 def _parse_email_list(raw) -> List[str]:
     """Parse emails from a list or comma-separated string."""
     if isinstance(raw, list):
@@ -373,17 +380,31 @@ class BaseEmailLogger(CustomLogger):
             greeting = html.escape(
                 event.user_email or event.key_alias or event.token or ""
             )
-            email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
-                email_logo_url=email_params.logo_url,
-                recipient_email=greeting,
-                percentage=percentage,
-                spend=spend_str,
-                max_budget=max_budget_str,
-                alert_threshold=alert_threshold_str,
-                base_url=email_params.base_url,
-                email_support_contact=email_params.support_contact,
-                email_footer=email_params.signature,
-            )
+            if event.event_group == Litellm_EntityType.TEAM_MEMBER:
+                email_html_content = TEAM_MEMBER_MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
+                    email_logo_url=email_params.logo_url,
+                    member=html.escape(event.user_email or event.user_id or ""),
+                    team_alias=html.escape(event.team_alias or event.team_id or ""),
+                    percentage=percentage,
+                    spend=spend_str,
+                    max_budget=max_budget_str,
+                    alert_threshold=alert_threshold_str,
+                    base_url=email_params.base_url,
+                    email_support_contact=email_params.support_contact,
+                    email_footer=email_params.signature,
+                )
+            else:
+                email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
+                    email_logo_url=email_params.logo_url,
+                    recipient_email=greeting,
+                    percentage=percentage,
+                    spend=spend_str,
+                    max_budget=max_budget_str,
+                    alert_threshold=alert_threshold_str,
+                    base_url=email_params.base_url,
+                    email_support_contact=email_params.support_contact,
+                    email_footer=email_params.signature,
+                )
             await self.send_email(
                 from_email=self.DEFAULT_LITELLM_EMAIL,
                 to_email=recipient_emails,
@@ -607,7 +628,7 @@ class BaseEmailLogger(CustomLogger):
             if user_info.spend < threshold_amount:
                 continue
 
-            _id = user_info.token or user_info.user_id or "default_id"
+            _id = _max_budget_alert_id(user_info)
             _cache_key = (
                 f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
             )
@@ -618,7 +639,7 @@ class BaseEmailLogger(CustomLogger):
                 emails.append(user_info.user_email)
             if not emails:
                 verbose_proxy_logger.warning(
-                    "No recipients for %d%% threshold on key %s, skipping alert",
+                    "No recipients for %d%% threshold on %s, skipping alert",
                     threshold_pct,
                     _id,
                 )
@@ -633,7 +654,11 @@ class BaseEmailLogger(CustomLogger):
             if send_count is not None and send_count > 1:
                 continue
 
-            event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
+            event_message = (
+                f"Team Member Budget Alert - {threshold_pct}% of Team Member Budget Reached"
+                if user_info.event_group == Litellm_EntityType.TEAM_MEMBER
+                else f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
+            )
             webhook_event = WebhookEvent(
                 event="max_budget_alert",
                 event_message=event_message,

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.constants import (
+    CLI_SESSION_KEY_PREFIX,
     MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
     MAX_OBJECTS_PER_POLL_CYCLE,
 )
@@ -147,10 +148,12 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
-    async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None:
+    async def _get_key_alias(self, batch_id: str, api_key: str | None, created_by: str | None) -> str | None:
         """Resolve the creating virtual key's alias from its hashed token."""
         if not api_key:
             return None
+        if created_by and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}":
+            return api_key
         try:
             key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
                 self.prisma_client
@@ -231,7 +234,7 @@ class CheckBatchCost:
             **(await self._get_user_info(batch_id, job.created_by)),
         }
 
-        key_alias = await self._get_key_alias(batch_id, api_key)
+        key_alias = await self._get_key_alias(batch_id, api_key, job.created_by)
         if key_alias is not None:
             metadata["user_api_key_alias"] = key_alias
         team_alias = await self._get_team_alias(team_id)

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -50,6 +50,7 @@ from litellm.proxy._types import (
     ProxyException,
     UserAPIKeyAuth,
 )
+from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
 from litellm.proxy.openai_files_endpoints.common_utils import (
     BATCH_CREATE_HIDDEN_PARAM,
     FILE_LIST_CONTINUATION_CHUNK_SIZE,
@@ -359,7 +360,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
         from prisma import Json
 
-        api_key = user_api_key_dict.api_key or None
+        api_key = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) or None
         attribution_columns = (
             {
                 **({"api_key": api_key} if api_key is not None else {}),

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.70"
+version = "0.1.71"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.70"
+version = "0.1.71"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_agent_kill_switch/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "kill_switch" JSONB;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -72,6 +72,7 @@ model LiteLLM_AgentsTable {
   agent_card_params Json
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
+  kill_switch          Json?
   agent_access_groups String[]       @default([])
   access_group_ids     String[]       @default([])
   object_permission_id String?

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.101"
+version = "0.4.102"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.101"
+version = "0.4.102"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/AGENTS.md

@@ -0,0 +1,22 @@
+# Rust workspace rules
+
+## Test placement
+
+- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
+- A test that reaches private items lives inline, in a `#[cfg(test)] mod tests { ... }` at the bottom of the file that owns those items
+- A test that only uses the crate's public API lives in `crates/<crate>/tests/<subject>.rs`, next to `src/`
+- Split a mixed test file along that line instead of widening visibility to move it
+- A test for another crate's item belongs in that crate, not in a downstream one
+- Never set `autotests = false` or hand-list `[[test]]` targets; every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own
+
+## Test fixtures and cases
+
+Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
+
+## Error definitions
+
+- A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`
+- Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each. A failure mode is something a caller handles differently (phase, status code, retry, a message Python parity pins exactly); failures no caller tells apart share one variant and differ only in its message
+- Wrap a lower-level error as a variant with `#[from]` or `#[source]` instead of flattening it to a string
+- Exception: split into separate types when different functions fail in disjoint ways, especially when different callers see them. A shared enum would force every caller to match variants its function can never return
+- Name a split type after what went wrong (a unit struct is fine for a single failure mode), not after the function that returns it

litellm-rust/Cargo.lock

@@ -73,6 +73,15 @@ version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
+[[package]]
+name = "arbitrary"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
+dependencies = [
+ "derive_arbitrary",
+]
+
 [[package]]
 name = "arc-swap"
 version = "1.9.2"
@@ -190,7 +199,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.0",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -701,14 +710,20 @@ dependencies = [
  "http 1.4.2",
  "http-body 1.1.0",
  "http-body-util",
+ "hyper 1.10.1",
+ "hyper-util",
  "itoa",
  "matchit",
  "memchr",
  "mime",
+ "multer",
  "percent-encoding",
  "pin-project-lite",
  "serde_core",
+ "serde_json",
+ "serde_path_to_error",
  "sync_wrapper",
+ "tokio",
  "tower",
  "tower-layer",
  "tower-service",
@@ -897,6 +912,12 @@ dependencies = [
  "hybrid-array",
 ]
 
+[[package]]
+name = "borrow-or-share"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
+
 [[package]]
 name = "bstr"
 version = "1.13.1"
@@ -914,6 +935,12 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -1032,18 +1059,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.6"
+version = "4.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
+checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.6.6"
+version = "4.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
+checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
 dependencies = [
  "anstyle",
  "clap_lex",
@@ -1159,7 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
 dependencies = [
  "digest 0.10.7",
- "spin",
+ "spin 0.10.1",
 ]
 
 [[package]]
@@ -1458,6 +1485,17 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "derive_arbitrary"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "derive_builder"
 version = "0.20.2"
@@ -1540,6 +1578,24 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
 
+[[package]]
+name = "email_address"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -1622,6 +1678,16 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
 
+[[package]]
+name = "filetime"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
+dependencies = [
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "find-msvc-tools"
 version = "0.1.9"
@@ -1639,6 +1705,17 @@ dependencies = [
  "zlib-rs",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e"
+dependencies = [
+ "borrow-or-share",
+ "ref-cast",
+ "serde",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1660,6 +1737,16 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fraction"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e246562084dde8ebbcc943b261c406ce4f68e5032ec28029a251a47d6a295500"
+dependencies = [
+ "num",
+ "num-bigint 0.4.8",
+]
+
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -1817,9 +1904,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "r-efi 5.3.0",
  "wasip2",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -2660,6 +2749,59 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonschema"
+version = "0.55.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b68339c3d874e48151d74ffe256d93a58cffa240983cb0967d3cbaea083a44fe"
+dependencies = [
+ "ahash",
+ "bytecount",
+ "data-encoding",
+ "email_address",
+ "fancy-regex 0.19.2",
+ "fraction",
+ "getrandom 0.3.4",
+ "itoa",
+ "jsonschema-regex",
+ "jsonschema-value",
+ "num-cmp",
+ "num-traits",
+ "percent-encoding",
+ "referencing",
+ "regex",
+ "serde",
+ "serde_json",
+ "strum",
+ "unicode-general-category",
+ "uuid-simd",
+]
+
+[[package]]
+name = "jsonschema-regex"
+version = "0.55.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6307b5b51216ec9b941b52244c74043fa0b1d6b657b56199f57cb1416d3641c5"
+dependencies = [
+ "regex-syntax",
+]
+
+[[package]]
+name = "jsonschema-value"
+version = "0.55.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0230ac05e09c6111e96c147b75c390579f5cbd45654b980c68ac60fe17b3f129"
+dependencies = [
+ "ahash",
+ "bytecount",
+ "fraction",
+ "getrandom 0.3.4",
+ "num-cmp",
+ "num-traits",
+ "serde_json",
+ "zmij",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -2689,6 +2831,10 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
+[[package]]
+name = "litellm"
+version = "0.0.1"
+
 [[package]]
 name = "litellm-auth"
 version = "0.1.0"
@@ -2713,6 +2859,7 @@ dependencies = [
  "litellm-http",
  "moka",
  "reqwest 0.12.28",
+ "rstest",
  "serde_json",
  "sha2 0.10.9",
  "thiserror 2.0.19",
@@ -2784,6 +2931,7 @@ dependencies = [
  "litellm-cache",
  "litellm-cache-response",
  "litellm-cache-testing",
+ "litellm-http",
  "reqwest 0.12.28",
  "rstest",
  "serde_json",
@@ -2817,6 +2965,7 @@ dependencies = [
  "litellm-auth-types",
  "litellm-cache",
  "litellm-cache-testing",
+ "litellm-http",
  "percent-encoding",
  "reqwest 0.12.28",
  "rstest",
@@ -2843,6 +2992,7 @@ dependencies = [
  "futures-util",
  "litellm-cache",
  "litellm-cache-testing",
+ "litellm-http",
  "qdrant-client",
  "reqwest 0.12.28",
  "rstest",
@@ -2916,6 +3066,7 @@ dependencies = [
  "litellm-auth-aws",
  "litellm-cache",
  "litellm-cache-testing",
+ "litellm-http",
  "reqwest 0.12.28",
  "rstest",
  "serde_json",
@@ -2960,6 +3111,18 @@ dependencies = [
  "strum",
 ]
 
+[[package]]
+name = "litellm-config"
+version = "0.1.0"
+dependencies = [
+ "litellm-auth-types",
+ "rstest",
+ "serde",
+ "serde_yaml_ng",
+ "tempfile",
+ "thiserror 2.0.19",
+]
+
 [[package]]
 name = "litellm-core"
 version = "0.1.0"
@@ -2975,6 +3138,7 @@ dependencies = [
  "litellm-http",
  "litellm-llms",
  "litellm-secrets",
+ "litellm-tracing",
  "litellm-types",
  "mime_guess",
  "moka",
@@ -2995,6 +3159,7 @@ dependencies = [
  "tokio-tungstenite",
  "url",
  "veil",
+ "wiremock",
 ]
 
 [[package]]
@@ -3009,10 +3174,20 @@ dependencies = [
  "serde_json",
  "serde_path_to_error",
  "serde_with",
+ "strum",
  "thiserror 2.0.19",
  "url",
 ]
 
+[[package]]
+name = "litellm-coroutine"
+version = "0.1.0"
+dependencies = [
+ "rstest",
+ "thiserror 2.0.19",
+ "tokio",
+]
+
 [[package]]
 name = "litellm-cost"
 version = "0.1.0"
@@ -3029,10 +3204,74 @@ dependencies = [
  "aws-smithy-types",
  "bytes",
  "futures-util",
+ "proptest",
  "rstest",
- "sse-stream",
  "thiserror 2.0.19",
  "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "litellm-gateway"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "futures-util",
+ "http-body-util",
+ "litellm-config",
+ "litellm-core",
+ "litellm-gateway-auth",
+ "litellm-gateway-inference",
+ "litellm-http",
+ "litellm-llms",
+ "litellm-secrets",
+ "litellm-tracing",
+ "rstest",
+ "serde_json",
+ "tokio",
+ "tower",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "litellm-gateway-auth"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "futures-util",
+ "litellm-auth-types",
+ "litellm-config",
+ "litellm-secrets",
+ "rstest",
+ "sha2 0.10.9",
+ "subtle",
+ "thiserror 2.0.19",
+ "tokio",
+ "tower",
+]
+
+[[package]]
+name = "litellm-gateway-inference"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "base64 0.22.1",
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-core",
+ "litellm-http",
+ "litellm-llms",
+ "litellm-router",
+ "litellm-secrets",
+ "litellm-types",
+ "rstest",
+ "serde_json",
+ "thiserror 2.0.19",
+ "tokio",
+ "tower",
+ "wiremock",
 ]
 
 [[package]]
@@ -3040,6 +3279,7 @@ name = "litellm-host"
 version = "0.1.0"
 dependencies = [
  "litellm-auth",
+ "litellm-coroutine",
  "rstest",
  "serde_json",
  "tokio",
@@ -3049,6 +3289,7 @@ dependencies = [
 name = "litellm-host-python"
 version = "0.1.0"
 dependencies = [
+ "bytes",
  "futures-util",
  "litellm-host",
  "pyo3",
@@ -3067,11 +3308,13 @@ dependencies = [
  "http 1.4.2",
  "hyper-util",
  "litellm-core-utils",
+ "rcgen",
  "reqwest 0.12.28",
  "rstest",
  "rustls 0.23.42",
  "serde",
  "serde_json",
+ "tempfile",
  "thiserror 2.0.19",
  "tokio",
  "veil",
@@ -3096,6 +3339,7 @@ dependencies = [
  "litellm-framing",
  "litellm-host",
  "litellm-http",
+ "litellm-python-compat",
  "litellm-secrets",
  "litellm-types",
  "reqwest 0.12.28",
@@ -3115,14 +3359,15 @@ dependencies = [
 name = "litellm-model-catalog"
 version = "0.1.0"
 dependencies = [
- "criterion",
  "indexmap 2.14.0",
- "litellm-model-catalog",
+ "jsonschema",
+ "litellm-types",
  "rstest",
  "schemars 1.2.2",
  "serde",
  "serde_json",
  "thiserror 2.0.19",
+ "time",
 ]
 
 [[package]]
@@ -3136,7 +3381,6 @@ dependencies = [
  "futures-util",
  "litellm-auth",
  "litellm-auth-aws",
- "litellm-auth-gcp",
  "litellm-cache",
  "litellm-cache-azure-blob",
  "litellm-cache-disk",
@@ -3171,6 +3415,7 @@ dependencies = [
  "serde_json",
  "serde_with",
  "sha2 0.10.9",
+ "strum",
  "thiserror 2.0.19",
  "tokio",
  "tokio-tungstenite",
@@ -3194,6 +3439,15 @@ dependencies = [
  "thiserror 2.0.19",
 ]
 
+[[package]]
+name = "litellm-router"
+version = "0.1.0"
+dependencies = [
+ "litellm-config",
+ "litellm-core",
+ "rstest",
+]
+
 [[package]]
 name = "litellm-secrets"
 version = "0.1.0"
@@ -3204,6 +3458,7 @@ dependencies = [
  "google-cloud-auth",
  "google-cloud-kms-v1",
  "litellm-core-utils",
+ "litellm-http",
  "litellm-python-compat",
  "litellm-secrets-aws",
  "litellm-secrets-azure",
@@ -3251,6 +3506,7 @@ dependencies = [
  "litellm-auth-azure",
  "litellm-auth-types",
  "litellm-core-utils",
+ "litellm-http",
  "litellm-secrets-types",
  "percent-encoding",
  "reqwest 0.12.28",
@@ -3270,6 +3526,7 @@ version = "0.1.0"
 dependencies = [
  "base64 0.22.1",
  "litellm-core-utils",
+ "litellm-http",
  "litellm-secrets-types",
  "litellm-tracing",
  "moka",
@@ -3298,6 +3555,7 @@ dependencies = [
  "litellm-auth-gcp",
  "litellm-auth-types",
  "litellm-core-utils",
+ "litellm-http",
  "litellm-secrets-types",
  "moka",
  "percent-encoding",
@@ -3339,11 +3597,33 @@ dependencies = [
  "rstest",
  "serde",
  "serde_json",
+ "strum",
  "thiserror 2.0.19",
  "tokio",
  "veil",
 ]
 
+[[package]]
+name = "litellm-testkit"
+version = "0.1.0"
+dependencies = [
+ "flate2",
+ "futures-util",
+ "reqwest 0.12.28",
+ "rstest",
+ "semver",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "tar",
+ "target-lexicon",
+ "tempfile",
+ "thiserror 2.0.19",
+ "tokio",
+ "toml",
+ "zip",
+]
+
 [[package]]
 name = "litellm-token-counter"
 version = "0.1.0"
@@ -3401,6 +3681,7 @@ dependencies = [
 name = "litellm-tracing"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "fancy-regex 0.19.2",
  "percent-encoding",
  "rstest",
@@ -3414,8 +3695,11 @@ dependencies = [
 name = "litellm-types"
 version = "0.1.0"
 dependencies = [
+ "rstest",
+ "schemars 1.2.2",
  "serde",
  "serde_json",
+ "strum",
 ]
 
 [[package]]
@@ -3492,6 +3776,12 @@ version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
+[[package]]
+name = "micromap"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74"
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -3577,6 +3867,23 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "multer"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
+dependencies = [
+ "bytes",
+ "encoding_rs",
+ "futures-util",
+ "http 1.4.2",
+ "httparse",
+ "memchr",
+ "mime",
+ "spin 0.9.9",
+ "version_check",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -3587,6 +3894,20 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "num"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
+dependencies = [
+ "num-bigint 0.4.8",
+ "num-complex",
+ "num-integer",
+ "num-iter",
+ "num-rational",
+ "num-traits",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.8"
@@ -3607,6 +3928,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -3631,6 +3958,27 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-iter"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint 0.4.8",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -4443,7 +4791,24 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.0",
+ "syn 3.0.6",
+]
+
+[[package]]
+name = "referencing"
+version = "0.55.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a196a5b4a8a12f46b6353174df865a05d41a6055aff212ec30877492788618b6"
+dependencies = [
+ "ahash",
+ "fluent-uri",
+ "getrandom 0.3.4",
+ "hashbrown 0.17.1",
+ "itoa",
+ "micromap",
+ "parking_lot",
+ "percent-encoding",
+ "serde_json",
 ]
 
 [[package]]
@@ -4905,7 +5270,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 3.0.0",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -4993,7 +5358,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.0",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -5004,7 +5369,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.0",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -5032,6 +5397,15 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -5075,6 +5449,19 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "serde_yaml_ng"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
+dependencies = [
+ "indexmap 2.14.0",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.7"
@@ -5204,6 +5591,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
+
 [[package]]
 name = "spin"
 version = "0.10.1"
@@ -5234,19 +5627,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "sse-stream"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
-dependencies = [
- "bytes",
- "futures-util",
- "http-body 1.1.0",
- "http-body-util",
- "pin-project-lite",
-]
-
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.1"
@@ -5316,9 +5696,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "3.0.0"
+version = "3.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5363,6 +5743,17 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
+[[package]]
+name = "tar"
+version = "0.4.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "target-lexicon"
 version = "0.13.5"
@@ -5419,7 +5810,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.0",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -5632,6 +6023,30 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml"
+version = "0.9.12+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
+dependencies = [
+ "indexmap 2.14.0",
+ "serde_core",
+ "serde_spanned",
+ "toml_datetime 0.7.5+spec-1.1.0",
+ "toml_parser",
+ "toml_writer",
+ "winnow 0.7.15",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.7.5+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "1.1.1+spec-1.1.0"
@@ -5648,9 +6063,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
 dependencies = [
  "indexmap 2.14.0",
- "toml_datetime",
+ "toml_datetime 1.1.1+spec-1.1.0",
  "toml_parser",
- "winnow",
+ "winnow 1.0.4",
 ]
 
 [[package]]
@@ -5659,9 +6074,15 @@ version = "1.1.3+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
 dependencies = [
- "winnow",
+ "winnow 1.0.4",
 ]
 
+[[package]]
+name = "toml_writer"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
+
 [[package]]
 name = "tonic"
 version = "0.14.6"
@@ -5929,6 +6350,12 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -5962,6 +6389,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -6009,6 +6442,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "uuid-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
 [[package]]
 name = "valuable"
 version = "0.1.1"
@@ -6407,6 +6850,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
+
 [[package]]
 name = "winnow"
 version = "1.0.4"
@@ -6469,6 +6918,16 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "xattr"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
+dependencies = [
+ "libc",
+ "rustix",
+]
+
 [[package]]
 name = "xmlparser"
 version = "0.13.6"
@@ -6594,6 +7053,23 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "zip"
+version = "2.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
+dependencies = [
+ "arbitrary",
+ "crc32fast",
+ "crossbeam-utils",
+ "displaydoc",
+ "flate2",
+ "indexmap 2.14.0",
+ "memchr",
+ "thiserror 2.0.19",
+ "zopfli",
+]
+
 [[package]]
 name = "zlib-rs"
 version = "0.6.7"
@@ -6605,3 +7081,15 @@ name = "zmij"
 version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zopfli"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "log",
+ "simd-adler32",
+]

litellm-rust/Cargo.toml

@@ -9,9 +9,14 @@ license = "MIT"
 repository = "https://github.com/BerriAI/litellm"
 
 [workspace.dependencies]
+litellm-config = { path = "crates/config" }
+litellm-router = { path = "crates/router" }
 litellm-tracing = { path = "crates/tracing" }
-tracing = "0.1"
 litellm-core = { path = "crates/core" }
+litellm-gateway = { path = "crates/gateway" }
+litellm-gateway-inference = { path = "crates/gateway-inference" }
+litellm-gateway-auth = { path = "crates/gateway-auth" }
+litellm-coroutine = { path = "crates/coroutine" }
 litellm-host = { path = "crates/host" }
 litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
 litellm-framing = { path = "crates/framer" }
@@ -47,7 +52,10 @@ litellm-token-counter-fast = { path = "crates/token-counter-fast" }
 litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
 litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
 litellm-host-python = { path = "crates/host-python" }
+litellm-python-compat = { path = "crates/python-compat" }
 
+tracing = "0.1"
+axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }
 bytes = "1"
 http = "1"
 google-cloud-auth = { version = "1.16.0", default-features = false }
@@ -56,8 +64,8 @@ hyper-util = { version = "0.1.20", default-features = false, features = ["client
 proptest = "1.7.0"
 pyo3 = "0.29.2"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
-pythonize = "0.29.0"
 rand = "0.8"
+schemars = "1"
 reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] }
 qdrant-client = { version = "1.19.0", default-features = false }
 uuid = { version = "1", features = ["v4"] }
@@ -80,6 +88,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
 base64 = "0.22"
+flate2 = "1"
+semver = "1"
+tar = "0.4"
+target-lexicon = "0.13.5"
+tempfile = "3"
+zip = { version = "2", default-features = false, features = ["deflate"] }
 moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"

litellm-rust/clippy.toml

@@ -7,4 +7,16 @@ disallowed-methods = [
     { path = "pyo3_async_runtimes::tokio::local_future_into_py", reason = "use litellm_host_python::run_async / run_async_value" },
     { path = "pyo3_async_runtimes::tokio::run", reason = "use litellm_host_python::run_sync / run_sync_value" },
     { path = "pyo3_async_runtimes::tokio::run_until_complete", reason = "use litellm_host_python::run_sync / run_sync_value" },
+    { path = "reqwest::Client::new", reason = "take litellm_http::Client from HttpClientPool" },
+    { path = "reqwest::Client::builder", reason = "HttpClientConfig owns client construction" },
+    { path = "reqwest::ClientBuilder::danger_accept_invalid_certs", reason = "set HttpClientConfig::verify instead" },
+    { path = "reqwest::ClientBuilder::identity", reason = "set HttpClientConfig::client_certificate instead" },
+    { path = "reqwest::ClientBuilder::use_preconfigured_tls", reason = "HttpClientConfig owns the TLS configuration" },
+]
+
+# Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS,
+# proxy and timeout settings. Only crates/http builds one.
+disallowed-types = [
+    { path = "reqwest::Client", reason = "take litellm_http::Client from HttpClientPool; only crates/http builds one" },
+    { path = "reqwest::ClientBuilder", reason = "HttpClientConfig owns client construction" },
 ]

litellm-rust/crates/auth-aws/Cargo.toml

@@ -22,5 +22,7 @@ aws-types = "1.4.0"
 aws-smithy-runtime-api = "1.13.0"
 
 [dev-dependencies]
+rstest.workspace = true
+litellm-http = { workspace = true, features = ["test-support"] }
 reqwest.workspace = true
 tokio.workspace = true

litellm-rust/crates/auth-aws/src/aws.rs

@@ -1,5 +1,4 @@
 use std::collections::BTreeMap;
-use std::sync::OnceLock;
 use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -26,8 +25,26 @@ use super::constants::{
 const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
 const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600);
 
-static STATIC_CREDENTIALS_CACHE: OnceLock<Cache<String, Credentials>> = OnceLock::new();
-static AMBIENT_CREDENTIALS_CACHE: OnceLock<Cache<String, Credentials>> = OnceLock::new();
+#[derive(Clone)]
+pub struct AwsAuthService {
+    static_credentials: Cache<String, Credentials>,
+    ambient_credentials: Cache<String, Credentials>,
+}
+
+impl Default for AwsAuthService {
+    fn default() -> Self {
+        Self {
+            static_credentials: Cache::builder()
+                .max_capacity(200)
+                .time_to_live(STATIC_CREDENTIALS_TTL)
+                .build(),
+            ambient_credentials: Cache::builder()
+                .max_capacity(200)
+                .time_to_live(AMBIENT_CREDENTIALS_TTL)
+                .build(),
+        }
+    }
+}
 
 fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option<Duration> {
     match flow {
@@ -108,35 +125,19 @@ fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String {
     format!("{:x}", hasher.finalize())
 }
 
-fn static_credentials_cache() -> &'static Cache<String, Credentials> {
-    STATIC_CREDENTIALS_CACHE.get_or_init(|| {
-        Cache::builder()
-            .max_capacity(200)
-            .time_to_live(STATIC_CREDENTIALS_TTL)
-            .build()
-    })
-}
+impl AwsAuthService {
+    fn get_cached_credentials(&self, key: &str) -> Option<Credentials> {
+        self.static_credentials
+            .get(key)
+            .or_else(|| self.ambient_credentials.get(key))
+    }
 
-fn ambient_credentials_cache() -> &'static Cache<String, Credentials> {
-    AMBIENT_CREDENTIALS_CACHE.get_or_init(|| {
-        Cache::builder()
-            .max_capacity(200)
-            .time_to_live(AMBIENT_CREDENTIALS_TTL)
-            .build()
-    })
-}
-
-fn get_cached_credentials(key: &str) -> Option<Credentials> {
-    static_credentials_cache()
-        .get(key)
-        .or_else(|| ambient_credentials_cache().get(key))
-}
-
-fn set_cached_credentials(key: String, credentials: Credentials, ttl: Duration) {
-    if ttl == STATIC_CREDENTIALS_TTL {
-        static_credentials_cache().insert(key, credentials);
-    } else {
-        ambient_credentials_cache().insert(key, credentials);
+    fn set_cached_credentials(&self, key: String, credentials: Credentials, ttl: Duration) {
+        if ttl == STATIC_CREDENTIALS_TTL {
+            self.static_credentials.insert(key, credentials);
+        } else {
+            self.ambient_credentials.insert(key, credentials);
+        }
     }
 }
 
@@ -214,66 +215,157 @@ pub fn classify_auth(
     AwsAuthFlow::DefaultChain
 }
 
-pub async fn resolve_credentials(
-    config: AwsAuthConfig,
-    env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-) -> Result<Credentials, Error> {
-    let resolved = config.clone().with_environment(env_lookup);
-    let flow = classify_auth(config, env_lookup);
-    match flow {
-        AwsAuthFlow::SessionToken {
-            access_key_id,
-            secret_access_key,
-            session_token,
-        } => Ok(Credentials::new(
-            access_key_id,
-            secret_access_key,
-            Some(session_token),
-            None,
-            "litellm-static-session",
-        )),
-        AwsAuthFlow::StaticKeys {
-            access_key_id,
-            secret_access_key,
-            region_name,
-        } => {
-            let flow = AwsAuthFlow::StaticKeys {
-                access_key_id: access_key_id.clone(),
-                secret_access_key: secret_access_key.clone(),
-                region_name,
-            };
-            let key = cache_key(&resolved, &flow);
-            if let Some(credentials) = get_cached_credentials(&key) {
-                return Ok(credentials);
-            }
-            let credentials = Credentials::new(
+impl AwsAuthService {
+    pub async fn resolve_credentials(
+        &self,
+        config: AwsAuthConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Credentials, Error> {
+        let resolved = config.clone().with_environment(env_lookup);
+        let flow = classify_auth(config, env_lookup);
+        match flow {
+            AwsAuthFlow::SessionToken {
                 access_key_id,
                 secret_access_key,
+                session_token,
+            } => Ok(Credentials::new(
+                access_key_id,
+                secret_access_key,
+                Some(session_token),
                 None,
-                None,
-                "litellm-static",
-            );
-            set_cached_credentials(
-                key,
-                credentials.clone(),
-                credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL),
-            );
-            Ok(credentials)
-        }
-        AwsAuthFlow::Profile { name } => {
-            let provider = aws_config::profile::ProfileFileCredentialsProvider::builder()
-                .profile_name(name)
-                .build();
-            provider
-                .provide_credentials()
-                .await
-                .map_err(|error| Error::AwsProfile(error.to_string()))
-        }
-        AwsAuthFlow::AssumeRole { role, session_name } => {
-            if is_already_running_as_role(&role, &resolved).await? {
-                let ambient_flow = AwsAuthFlow::DefaultChain;
-                let key = cache_key(&resolved, &ambient_flow);
-                if let Some(credentials) = get_cached_credentials(&key) {
+                "litellm-static-session",
+            )),
+            AwsAuthFlow::StaticKeys {
+                access_key_id,
+                secret_access_key,
+                region_name,
+            } => {
+                let flow = AwsAuthFlow::StaticKeys {
+                    access_key_id: access_key_id.clone(),
+                    secret_access_key: secret_access_key.clone(),
+                    region_name,
+                };
+                let key = cache_key(&resolved, &flow);
+                if let Some(credentials) = self.get_cached_credentials(&key) {
+                    return Ok(credentials);
+                }
+                let credentials = Credentials::new(
+                    access_key_id,
+                    secret_access_key,
+                    None,
+                    None,
+                    "litellm-static",
+                );
+                self.set_cached_credentials(
+                    key,
+                    credentials.clone(),
+                    credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL),
+                );
+                Ok(credentials)
+            }
+            AwsAuthFlow::Profile { name } => {
+                let provider = aws_config::profile::ProfileFileCredentialsProvider::builder()
+                    .profile_name(name)
+                    .build();
+                provider
+                    .provide_credentials()
+                    .await
+                    .map_err(|error| Error::AwsProfile(error.to_string()))
+            }
+            AwsAuthFlow::AssumeRole { role, session_name } => {
+                if is_already_running_as_role(&role, &resolved).await? {
+                    let ambient_flow = AwsAuthFlow::DefaultChain;
+                    let key = cache_key(&resolved, &ambient_flow);
+                    if let Some(credentials) = self.get_cached_credentials(&key) {
+                        return Ok(credentials);
+                    }
+                    let provider =
+                    aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
+                        .build()
+                        .await;
+                    let credentials = provider
+                        .provide_credentials()
+                        .await
+                        .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
+                    self.set_cached_credentials(
+                        key,
+                        credentials.clone(),
+                        credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
+                    );
+                    return Ok(credentials);
+                }
+                let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
+                if let Some(region) = resolved.region_name.clone() {
+                    loader = loader.region(aws_types::region::Region::new(region));
+                }
+                if let Some(endpoint) = resolved.sts_endpoint.clone() {
+                    loader = loader.endpoint_url(endpoint);
+                }
+                if let (Some(access_key_id), Some(secret_access_key)) =
+                    (resolved.access_key_id, resolved.secret_access_key)
+                {
+                    loader = loader.credentials_provider(Credentials::new(
+                        access_key_id,
+                        secret_access_key,
+                        resolved.session_token,
+                        None,
+                        "litellm-role-source",
+                    ));
+                }
+                let sdk_config = loader.load().await;
+                let builder = aws_config::sts::AssumeRoleProvider::builder(role);
+                let builder = match session_name {
+                    Some(name) => builder.session_name(name),
+                    None => builder.session_name(default_session_name()),
+                };
+                let builder = match resolved.external_id {
+                    Some(id) => builder.external_id(id),
+                    None => builder,
+                };
+                let provider = builder.configure(&sdk_config).build().await;
+                provider
+                    .provide_credentials()
+                    .await
+                    .map_err(|error| Error::AwsAssumeRole(error.to_string()))
+            }
+            AwsAuthFlow::WebIdentity {
+                token,
+                role,
+                session_name,
+            } => {
+                let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
+                if let Some(region) = resolved.region_name {
+                    loader = loader.region(aws_types::region::Region::new(region));
+                }
+                if let Some(endpoint) = resolved.sts_endpoint {
+                    loader = loader.endpoint_url(endpoint);
+                }
+                let sdk_config = loader.load().await;
+                let client = aws_sdk_sts::Client::new(&sdk_config);
+                let response = client
+                    .assume_role_with_web_identity()
+                    .role_arn(role)
+                    .role_session_name(session_name)
+                    .web_identity_token(token)
+                    .send()
+                    .await
+                    .map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
+                let credentials = response
+                    .credentials()
+                    .ok_or(Error::AwsMissingWebIdentityCredentials)?;
+                let expiration = SystemTime::try_from(*credentials.expiration())
+                    .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
+                Ok(Credentials::new(
+                    credentials.access_key_id(),
+                    credentials.secret_access_key(),
+                    Some(credentials.session_token().to_string()),
+                    Some(expiration),
+                    "litellm-web-identity",
+                ))
+            }
+            AwsAuthFlow::DefaultChain => {
+                let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
+                if let Some(credentials) = self.get_cached_credentials(&key) {
                     return Ok(credentials);
                 }
                 let provider =
@@ -284,101 +376,14 @@ pub async fn resolve_credentials(
                     .provide_credentials()
                     .await
                     .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
-                set_cached_credentials(
+                self.set_cached_credentials(
                     key,
                     credentials.clone(),
-                    credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
+                    credential_cache_ttl(&AwsAuthFlow::DefaultChain)
+                        .unwrap_or(AMBIENT_CREDENTIALS_TTL),
                 );
-                return Ok(credentials);
+                Ok(credentials)
             }
-            let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
-            if let Some(region) = resolved.region_name.clone() {
-                loader = loader.region(aws_types::region::Region::new(region));
-            }
-            if let Some(endpoint) = resolved.sts_endpoint.clone() {
-                loader = loader.endpoint_url(endpoint);
-            }
-            if let (Some(access_key_id), Some(secret_access_key)) =
-                (resolved.access_key_id, resolved.secret_access_key)
-            {
-                loader = loader.credentials_provider(Credentials::new(
-                    access_key_id,
-                    secret_access_key,
-                    resolved.session_token,
-                    None,
-                    "litellm-role-source",
-                ));
-            }
-            let sdk_config = loader.load().await;
-            let builder = aws_config::sts::AssumeRoleProvider::builder(role);
-            let builder = match session_name {
-                Some(name) => builder.session_name(name),
-                None => builder.session_name(default_session_name()),
-            };
-            let builder = match resolved.external_id {
-                Some(id) => builder.external_id(id),
-                None => builder,
-            };
-            let provider = builder.configure(&sdk_config).build().await;
-            provider
-                .provide_credentials()
-                .await
-                .map_err(|error| Error::AwsAssumeRole(error.to_string()))
-        }
-        AwsAuthFlow::WebIdentity {
-            token,
-            role,
-            session_name,
-        } => {
-            let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
-            if let Some(region) = resolved.region_name {
-                loader = loader.region(aws_types::region::Region::new(region));
-            }
-            if let Some(endpoint) = resolved.sts_endpoint {
-                loader = loader.endpoint_url(endpoint);
-            }
-            let sdk_config = loader.load().await;
-            let client = aws_sdk_sts::Client::new(&sdk_config);
-            let response = client
-                .assume_role_with_web_identity()
-                .role_arn(role)
-                .role_session_name(session_name)
-                .web_identity_token(token)
-                .send()
-                .await
-                .map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
-            let credentials = response
-                .credentials()
-                .ok_or(Error::AwsMissingWebIdentityCredentials)?;
-            let expiration = SystemTime::try_from(*credentials.expiration())
-                .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
-            Ok(Credentials::new(
-                credentials.access_key_id(),
-                credentials.secret_access_key(),
-                Some(credentials.session_token().to_string()),
-                Some(expiration),
-                "litellm-web-identity",
-            ))
-        }
-        AwsAuthFlow::DefaultChain => {
-            let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
-            if let Some(credentials) = get_cached_credentials(&key) {
-                return Ok(credentials);
-            }
-            let provider =
-                aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
-                    .build()
-                    .await;
-            let credentials = provider
-                .provide_credentials()
-                .await
-                .map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
-            set_cached_credentials(
-                key,
-                credentials.clone(),
-                credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL),
-            );
-            Ok(credentials)
         }
     }
 }
@@ -585,6 +590,37 @@ pub fn aws_auth_config(
     }
 }
 
+/// Where the credentials that sign a request come from, decided when the request is
+/// prepared and resolved when it is sent.
+#[derive(Clone, Debug, PartialEq)]
+pub enum AwsCredentialSource {
+    HostSupplied(Credentials),
+    Chain(AwsAuthConfig),
+}
+
+impl AwsCredentialSource {
+    pub fn from_params(
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Self {
+        match host_supplied_credentials(optional_params) {
+            Some(credentials) => Self::HostSupplied(credentials),
+            None => Self::Chain(aws_auth_config(optional_params, env_lookup)),
+        }
+    }
+
+    pub async fn resolve(
+        self,
+        auth: &AwsAuthService,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Credentials, Error> {
+        match self {
+            Self::HostSupplied(credentials) => Ok(credentials),
+            Self::Chain(config) => auth.resolve_credentials(config, env_lookup).await,
+        }
+    }
+}
+
 /// Credentials a host resolved through its own chain and handed down verbatim.
 ///
 /// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads
@@ -747,17 +783,18 @@ mod tests {
 
     #[tokio::test]
     async fn static_credentials_do_not_use_network() {
-        let credentials = resolve_credentials(
-            AwsAuthConfig {
-                access_key_id: Some("ak".into()),
-                secret_access_key: Some("sk".into()),
-                region_name: Some("us-east-1".into()),
-                ..Default::default()
-            },
-            &no_env,
-        )
-        .await
-        .expect("static credentials");
+        let credentials = AwsAuthService::default()
+            .resolve_credentials(
+                AwsAuthConfig {
+                    access_key_id: Some("ak".into()),
+                    secret_access_key: Some("sk".into()),
+                    region_name: Some("us-east-1".into()),
+                    ..Default::default()
+                },
+                &no_env,
+            )
+            .await
+            .expect("static credentials");
         assert_eq!(credentials.access_key_id(), "ak");
         assert_eq!(credentials.session_token(), None);
     }
@@ -807,17 +844,67 @@ mod tests {
         );
     }
 
-    #[test]
+    #[rstest::rstest]
     fn cache_round_trip_preserves_credentials() {
+        let auth = AwsAuthService::default();
         let key = format!("cache-test-{}", std::process::id());
         let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test");
-        set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL);
+        auth.set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL);
         assert_eq!(
-            get_cached_credentials(&key).map(|value| value.access_key_id().to_string()),
+            auth.get_cached_credentials(&key)
+                .map(|value| value.access_key_id().to_string()),
             Some("cache-ak".to_string())
         );
     }
 
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn cloned_services_reuse_credentials_but_independent_services_do_not() {
+        let auth = AwsAuthService::default();
+        let config = AwsAuthConfig {
+            access_key_id: Some("configured-key".into()),
+            secret_access_key: Some("configured-secret".into()),
+            region_name: Some("us-east-1".into()),
+            ..AwsAuthConfig::default()
+        };
+        let flow = classify_auth(config.clone(), &no_env);
+        let cached = Credentials::new("cached-key", "cached-secret", None, None, "test");
+        auth.set_cached_credentials(
+            cache_key(&config, &flow),
+            cached.clone(),
+            STATIC_CREDENTIALS_TTL,
+        );
+
+        let reused = auth
+            .clone()
+            .resolve_credentials(config.clone(), &no_env)
+            .await
+            .unwrap();
+        let independent = AwsAuthService::default()
+            .resolve_credentials(config.clone(), &no_env)
+            .await
+            .unwrap();
+        let different = AwsAuthConfig {
+            access_key_id: Some("different-key".into()),
+            ..config.clone()
+        };
+        let other_identity = auth
+            .resolve_credentials(different.clone(), &no_env)
+            .await
+            .unwrap();
+
+        assert_eq!(reused.access_key_id(), cached.access_key_id());
+        assert_eq!(reused.secret_access_key(), cached.secret_access_key());
+        assert_eq!(
+            Some(independent.access_key_id()),
+            config.access_key_id.as_deref()
+        );
+        assert_eq!(
+            Some(other_identity.access_key_id()),
+            different.access_key_id.as_deref()
+        );
+    }
+
     #[test]
     fn same_role_comparison_matches_partition_account_and_role() {
         assert!(same_role_arns(
@@ -952,17 +1039,18 @@ mod tests {
         let body = br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":1,"messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}"#.to_vec();
         let headers =
             BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]);
-        let credentials = resolve_credentials(
-            AwsAuthConfig {
-                access_key_id: Some(access_key_id),
-                secret_access_key: Some(secret_access_key),
-                region_name: Some("us-west-2".to_string()),
-                ..Default::default()
-            },
-            &no_env,
-        )
-        .await?;
-        let client = reqwest::Client::new();
+        let credentials = AwsAuthService::default()
+            .resolve_credentials(
+                AwsAuthConfig {
+                    access_key_id: Some(access_key_id),
+                    secret_access_key: Some(secret_access_key),
+                    region_name: Some("us-west-2".to_string()),
+                    ..Default::default()
+                },
+                &no_env,
+            )
+            .await?;
+        let client = litellm_http::Client::plain_for_test();
         let mut failures = Vec::new();
 
         for region in ["us-west-2", "us-east-1"] {

litellm-rust/crates/auth-aws/src/signer.rs

@@ -1,13 +1,11 @@
 use std::{collections::BTreeMap, time::SystemTime};
 
+use crate::{
+    AwsAuthService, AwsCredentialSource, Error, aws_signature_headers, is_sigv4_computed_header,
+    sign_post,
+};
 use aws_credential_types::Credentials;
 use litellm_http::outbound::{RequestSigner, UnsignedRequest};
-use serde_json::{Map, Value};
-
-use crate::{
-    Error, aws_auth_config, aws_signature_headers, host_supplied_credentials,
-    is_sigv4_computed_header, resolve_credentials, sign_post,
-};
 
 #[derive(Clone, Debug)]
 pub struct SigV4Signer {
@@ -32,19 +30,17 @@ impl SigV4Signer {
     }
 
     pub async fn resolve(
+        auth: &AwsAuthService,
         region: String,
         service: &'static str,
-        optional_params: &Map<String, Value>,
+        credentials: AwsCredentialSource,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Self, Error> {
-        let credentials = match host_supplied_credentials(optional_params) {
-            Some(credentials) => credentials,
-            None => {
-                resolve_credentials(aws_auth_config(optional_params, env_lookup), env_lookup)
-                    .await?
-            }
-        };
-        Ok(Self::new(region, service, credentials))
+        Ok(Self::new(
+            region,
+            service,
+            credentials.resolve(auth, env_lookup).await?,
+        ))
     }
 }
 
@@ -80,7 +76,7 @@ mod tests {
     use std::time::{Duration, UNIX_EPOCH};
 
     use litellm_http::outbound::OutboundRequest;
-    use serde_json::json;
+    use serde_json::{Value, json};
 
     use super::*;
 

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -131,7 +131,7 @@ impl Default for VertexAuth {
 }
 
 impl VertexAuth {
-    fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
+    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
         Self {
             providers: Cache::builder().max_capacity(64).build(),
             loader,
@@ -220,16 +220,16 @@ impl VertexAuth {
     }
 }
 
-trait VertexTokenSource: Send + Sync {
+pub trait VertexTokenSource: Send + Sync {
     fn project_id(&self) -> VertexAuthFuture<'_, String>;
     fn token(&self) -> VertexAuthFuture<'_, String>;
 }
 
-trait VertexProviderLoader: Send + Sync {
+pub trait VertexProviderLoader: Send + Sync {
     fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
 }
 
-type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
 
 struct GcpTokenSource(Arc<dyn TokenProvider>);
 
@@ -305,7 +305,7 @@ fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
 }
 
 #[derive(Clone, Debug)]
-enum CredentialSource {
+pub enum CredentialSource {
     Inline(SecretValue),
     Trusted(SecretValue),
     ApplicationCredentials(String),

litellm-rust/crates/auth-types/src/http.rs

@@ -7,7 +7,7 @@ pub enum CredentialPlacement {
 }
 
 impl CredentialPlacement {
-    pub fn header_name(self) -> &'static str {
+    pub const fn header_name(self) -> &'static str {
         match self {
             Self::Bearer => "Authorization",
             Self::Header(name) => name,
@@ -40,21 +40,6 @@ pub fn apply_credential(
     )
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum RequestAuth {
-    Header {
-        name: &'static str,
-        value: String,
-    },
-    Bearer {
-        token: String,
-    },
-    AwsSigV4 {
-        region: String,
-        service: &'static str,
-    },
-}
-
 #[cfg(test)]
 mod tests {
     use super::{CredentialPlacement, apply_credential};

litellm-rust/crates/auth-types/src/lib.rs

@@ -51,7 +51,7 @@ pub use credential::{
     CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
 };
 pub use error::Error;
-pub use http::{CredentialPlacement, RequestAuth};
+pub use http::CredentialPlacement;
 pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
 pub use secret::SecretValue;
 pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

litellm-rust/crates/auth/src/lib.rs

@@ -2,6 +2,9 @@
 
 pub use litellm_auth_types::*;
 
+mod services;
+pub use services::AuthServices;
+
 #[cfg(feature = "aws")]
 pub use litellm_auth_aws as aws;
 #[cfg(feature = "azure")]

litellm-rust/crates/auth/src/services.rs

@@ -0,0 +1,9 @@
+#[derive(Default)]
+pub struct AuthServices {
+    #[cfg(feature = "aws")]
+    pub aws: litellm_auth_aws::AwsAuthService,
+    #[cfg(feature = "azure")]
+    pub azure: litellm_auth_azure::AzureAuthService,
+    #[cfg(feature = "gcp")]
+    pub gcp: litellm_auth_gcp::VertexAuth,
+}

litellm-rust/crates/cache-azure-blob/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+litellm-http.workspace = true
 litellm-auth-azure.workspace = true
 litellm-auth-types.workspace = true
 litellm-cache.workspace = true
@@ -19,6 +20,7 @@ tokio.workspace = true
 url.workspace = true
 
 [dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
 litellm-cache-response.workspace = true
 litellm-cache-testing.workspace = true
 rstest.workspace = true

litellm-rust/crates/cache-azure-blob/src/cache.rs

@@ -31,7 +31,7 @@ impl<C: CacheCodec> AzureBlobCache<C> {
     pub async fn connect(
         account_url: &str,
         container: &str,
-        http: reqwest::Client,
+        http: litellm_http::Client,
         codec: C,
         runtime: Handle,
     ) -> Result<Self, Error> {

litellm-rust/crates/cache-azure-blob/src/transport.rs

@@ -8,7 +8,7 @@ use azure_core::{
 use futures_util::TryStreamExt;
 
 #[derive(Debug)]
-pub struct ReqwestTransport(pub reqwest::Client);
+pub struct ReqwestTransport(pub litellm_http::Client);
 
 #[async_trait::async_trait]
 impl HttpClient for ReqwestTransport {

litellm-rust/crates/cache-azure-blob/tests/transport.rs

@@ -31,7 +31,7 @@ async fn connect(server: &MockServer) -> AzureBlobCache<JsonCodec<Value>> {
         None,
         ClientOptions {
             transport: Some(Transport::new(Arc::new(ReqwestTransport(
-                reqwest::Client::new(),
+                litellm_http::Client::plain_for_test(),
             )))),
             ..ClientOptions::default()
         },

litellm-rust/crates/cache-gcs/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+litellm-http.workspace = true
 futures-util.workspace = true
 litellm-auth-gcp.workspace = true
 litellm-auth-types.workspace = true
@@ -15,6 +16,7 @@ reqwest.workspace = true
 tokio.workspace = true
 
 [dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
 litellm-cache-testing.workspace = true
 rstest.workspace = true
 serde_json.workspace = true

litellm-rust/crates/cache-gcs/src/cache.rs

@@ -5,8 +5,8 @@ use litellm_cache::{
     BaseCache, BatchCache, BatchEntry, CacheCodec, DisconnectCache, Error, ExactCacheContext,
     FlushCache,
 };
+use litellm_http::Client;
 use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
-use reqwest::Client;
 
 use crate::{GcpTokenSource, TokenSource};
 

litellm-rust/crates/cache-gcs/tests/cache.rs

@@ -96,7 +96,7 @@ async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) {
             path_service_account: Some("/secrets/sa.json".into()),
             ..support::config(&server, Some("folder"))
         },
-        reqwest::Client::new(),
+        litellm_http::Client::plain_for_test(),
         litellm_cache::JsonCodec::<Value>::new(),
     );
     assert_eq!(cache.bucket_name(), "bucket");

litellm-rust/crates/cache-gcs/tests/support/mod.rs

@@ -29,7 +29,7 @@ pub fn cache_with_token(
 ) -> JsonGcsCache {
     GcsCache::with_token_source(
         config(server, gcs_path),
-        reqwest::Client::new(),
+        litellm_http::Client::plain_for_test(),
         JsonCodec::new(),
         token,
     )

litellm-rust/crates/cache-qdrant-semantic/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+litellm-http.workspace = true
 futures-util.workspace = true
 litellm-cache.workspace = true
 qdrant-client = { workspace = true, features = ["serde"] }
@@ -17,6 +18,7 @@ tokio.workspace = true
 uuid.workspace = true
 
 [dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
 futures-executor = "0.3"
 litellm-cache-testing.workspace = true
 rstest.workspace = true

litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs

@@ -1,7 +1,7 @@
 use std::time::Duration;
 
 use litellm_cache::{Error, semantic::Embedder};
-use reqwest::Client;
+use litellm_http::Client;
 use serde_json::Value;
 
 pub struct OpenAiEmbedder {

litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs

@@ -5,6 +5,10 @@ use std::{
 
 use litellm_cache::{Error, semantic::Embedder};
 use litellm_cache_qdrant_semantic::{OpenAiEmbedder, OpenAiEmbedderConfig};
+use litellm_http::{
+    ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, Resolution,
+    media::PublicDnsResolver,
+};
 use rstest::rstest;
 use serde_json::{Value, json};
 use tokio::{
@@ -104,7 +108,7 @@ fn config(base: String, timeout: Option<Duration>) -> OpenAiEmbedderConfig {
 async fn posts_embeddings_request_and_parses_vector() {
     let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
     let embedder = OpenAiEmbedder::new(
-        reqwest::Client::new(),
+        litellm_http::Client::plain_for_test(),
         config(
             format!("{}/", server.base_url()),
             Some(Duration::from_secs(1)),
@@ -156,14 +160,17 @@ async fn status_timeout_and_body_errors_are_unavailable(
 ) {
     let server =
         TestHttpServer::response_after(status, body, Duration::from_millis(delay_ms)).await;
-    let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), timeout));
+    let embedder = OpenAiEmbedder::new(
+        litellm_http::Client::plain_for_test(),
+        config(server.base_url(), timeout),
+    );
     assert_eq!(embedder.async_embed("hello", None).await, expected);
 }
 
 #[rstest]
 fn sync_embedding_is_unsupported() {
     let embedder = OpenAiEmbedder::new(
-        reqwest::Client::new(),
+        litellm_http::Client::plain_for_test(),
         config("http://127.0.0.1:9".to_owned(), None),
     );
     assert_eq!(
@@ -176,9 +183,12 @@ fn sync_embedding_is_unsupported() {
 #[tokio::test]
 async fn uses_the_injected_client() {
     let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
-    let client = reqwest::Client::builder()
-        .user_agent("litellm-embedder-test")
-        .build()
+    let config_with_agent = HttpClientConfig {
+        user_agent: Some("litellm-embedder-test".into()),
+        ..Resolution::from(&HttpSettings::default()).config
+    };
+    let client = HttpClientPool::new(Arc::new(PublicDnsResolver))
+        .client(&config_with_agent, ClientVariant::Provider)
         .unwrap();
     let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None));
     assert_eq!(

litellm-rust/crates/cache-s3/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+litellm-http.workspace = true
 litellm-cache.workspace = true
 litellm-auth-aws.workspace = true
 aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
@@ -19,6 +20,7 @@ reqwest.workspace = true
 tokio.workspace = true
 
 [dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
 litellm-cache-testing.workspace = true
 rstest.workspace = true
 wiremock = "0.6.5"

litellm-rust/crates/cache-s3/src/auth.rs

@@ -2,10 +2,11 @@ use aws_credential_types::{
     Credentials as AwsCredentials,
     provider::{ProvideCredentials, error::CredentialsError, future},
 };
-use litellm_auth_aws::{AwsAuthConfig, resolve_credentials};
+use litellm_auth_aws::{AwsAuthConfig, AwsAuthService};
 
 #[derive(Clone)]
 pub struct S3Credentials {
+    auth: AwsAuthService,
     config: AwsAuthConfig,
     env: fn(&str) -> Option<String>,
 }
@@ -16,7 +17,11 @@ impl S3Credentials {
     }
 
     pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option<String>) -> Self {
-        Self { config, env }
+        Self {
+            auth: AwsAuthService::default(),
+            config,
+            env,
+        }
     }
 }
 
@@ -38,7 +43,8 @@ impl ProvideCredentials for S3Credentials {
                     "litellm-s3-cache",
                 ));
             }
-            resolve_credentials(self.config.clone(), &self.env)
+            self.auth
+                .resolve_credentials(self.config.clone(), &self.env)
                 .await
                 .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed"))
         })

litellm-rust/crates/cache-s3/src/cache.rs

@@ -42,7 +42,12 @@ pub struct S3Cache<C: CacheCodec> {
 }
 
 impl<C: CacheCodec> S3Cache<C> {
-    pub fn new(config: S3CacheConfig, http: reqwest::Client, codec: C, runtime: Handle) -> Self {
+    pub fn new(
+        config: S3CacheConfig,
+        http: litellm_http::Client,
+        codec: C,
+        runtime: Handle,
+    ) -> Self {
         let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
         let base = aws_sdk_s3::Config::builder()
             .behavior_version(BehaviorVersion::latest())

litellm-rust/crates/cache-s3/src/transport.rs

@@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{
 use aws_smithy_types::body::SdkBody;
 
 #[derive(Clone, Debug)]
-pub(crate) struct ReqwestHttpClient(pub(crate) reqwest::Client);
+pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client);
 
 impl HttpClient for ReqwestHttpClient {
     fn http_connector(

litellm-rust/crates/cache-s3/tests/support/mod.rs

@@ -32,7 +32,12 @@ pub fn config(endpoint: &str) -> S3CacheConfig {
 }
 
 pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache {
-    S3Cache::new(config, reqwest::Client::new(), JsonCodec::new(), runtime)
+    S3Cache::new(
+        config,
+        litellm_http::Client::plain_for_test(),
+        JsonCodec::new(),
+        runtime,
+    )
 }
 
 pub fn cache(endpoint: &str) -> JsonS3Cache {

litellm-rust/crates/callbacks-legacy-python/AGENTS.md

@@ -1,19 +1,19 @@
 - Target invariants, not completion claims
-- Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits)
+- This crate is the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on)
+  - Smell test: if a future callback host (`callbacks-v1-python`, WASM, in-process Rust) could share a piece of this crate, it does not belong here
+  - SDK request policy (credential inheritance, the budget and retry-count limits) is the driver's preflight, supplied by `python-bridge`; this crate only adopts the keyword view it produces
   - The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call
-- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`)
+- Every litellm Python internal Rust still borrows is a variant of `LegacyPython`, grouped by subsystem, with its signature pinned in `python_contract.json`
   - The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it
   - Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython`
-  - `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_call` and keep no copy
-- `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does
-  - Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's
+- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the call rewrites it (setup, deployment hook, preflight) and the bound request object backing omitted keywords; routes hand it over through `run_legacy_call` and keep no copy
+- `setup` reuses a `Logging` passed as `litellm_logging_obj` (the proxy and Router) and otherwise builds one through `function_setup`; which callbacks run is `Logging`'s decision, never this crate's
 - Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation
-  - Retain complete boundary arguments, opaque unknown values, aliases, omitted/default distinctions and deliberate copies; preserve the established deployment-hook kwargs view
-  - Before `pre_call`, re-alias every body key whose value equals the caller's argument to the caller's own object; this crate compares the two itself, and the argument is resolved by `litellm_host_python::lookup`
-  - Retain independently captured body/header roots from `pre_call` to `post_call`; in-place mutation reaches the wire, envelope field replacement is visible to later callbacks only
-  - A later kind of callback host (WASM, in-process Rust) has none of these obligations, so they stay out of `litellm-host`, `litellm-host-python` and the bridge; the only fact that crosses from the route is the prepared keyword view
-- Success and failure handlers receive the exact selected public response or exception; logging projections, redaction and snapshots keep their own copy contracts
-  - Ordinary failure-handler errors cannot suppress the other eligible family or replace the mapped provider error; a cancellation ends the call with no further dispatch
-  - Dispatch errors never replay provider work or trigger the opposite outcome; the proxy's acceptance or rejection releases deferred success at most once
+  - Retain complete boundary arguments, opaque values, aliases, omitted/default distinctions and deliberate copies; preserve the deployment-hook kwargs view
+  - Before `pre_call`, re-alias every body key whose value equals the caller's argument to the caller's own object, resolved through `litellm_host_python::lookup`
+  - Retain body/header roots from `pre_call` to `post_call`; in-place mutation reaches the wire, envelope field replacement is visible to later callbacks only
+- Success and failure handlers receive the exact selected public response or exception
+  - A failure-handler error cannot suppress the other eligible family or replace the mapped provider error; a cancellation ends the call with no further dispatch
+  - Dispatch errors never replay provider work or trigger the opposite outcome; the proxy releases deferred success at most once
   - Delivery follows the registry, not the callable's type: direct, awaited, executor-submitted, logging-worker and deferred paths stay distinct
 - Traverse every retained Python edge; `close` is idempotent and restores the correlation context once

litellm-rust/crates/callbacks-legacy-python/Cargo.toml

@@ -4,7 +4,6 @@ version = "0.1.0"
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-autotests = false
 
 [dependencies]
 litellm-host.workspace = true

litellm-rust/crates/callbacks-legacy-python/python_contract.json

@@ -6,9 +6,6 @@
     "start_time",
     "asynchronous"
   ],
-  "check_limits": [
-    "kwargs"
-  ],
   "finalize": [
     "response",
     "logger",
@@ -76,11 +73,6 @@
   ],
   "custom_pricing_fields": [],
   "is_internal_call": [],
-  "credential_list": [],
-  "warn_unknown_credential": [
-    "name",
-    "loaded"
-  ],
   "before_deployment_call": [
     "kwargs",
     "call_type"

litellm-rust/crates/callbacks-legacy-python/src/call.rs

@@ -3,8 +3,8 @@
 //! lifetime. No other callback host has that obligation, which is why nothing outside
 //! this crate holds them.
 
-use litellm_host::{machine::Machine, route::Route};
-use litellm_host_python::{RouteHost, lookup, run_call};
+use litellm_host::{machine::Machine, protocol::Protocol};
+use litellm_host_python::{Preflight, ProtocolHost, lookup, run_call};
 use pyo3::{
     gc::{PyTraverseError, PyVisit},
     prelude::*,
@@ -39,7 +39,8 @@ impl PublicCall {
     }
 
     /// The keyword view the legacy path currently reads: the caller's copy until
-    /// `function_setup`, then each rewrite (setup, deployment hook, prepare) in turn.
+    /// `function_setup`, then each rewrite (setup, deployment hook, the driver's preflight)
+    /// in turn.
     pub(crate) fn kwargs(&self) -> &Py<PyDict> {
         &self.kwargs
     }
@@ -63,26 +64,29 @@ impl PublicCall {
     }
 }
 
-/// Runs one native call under the legacy `Logging` contract: the route host projects from
-/// the keyword view the contract prepares, and the contract observes the call.
+/// Runs one native call under the legacy `Logging` contract: the protocol host projects from
+/// the keyword view the contract prepares and `preflight` rewrites, and the contract
+/// observes the call.
 pub fn run_legacy_call<H, M>(
     py: Python<'_>,
     surface: LegacySurface,
     call: PublicCall,
     machine: M,
-    route: H,
+    host: H,
+    preflight: Preflight,
     asynchronous: bool,
 ) -> PyResult<Py<PyAny>>
 where
-    H: RouteHost + 'static,
-    M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
+    H: ProtocolHost + 'static,
+    M: Machine<Protocol = H::Protocol, Complete = <H::Protocol as Protocol>::Response> + 'static,
 {
     let arguments = call.kwargs.clone_ref(py);
     run_call(
         py,
         machine,
-        route,
+        host,
         Box::new(LegacyLogging::new(py, surface, call, asynchronous)),
+        preflight,
         arguments,
         asynchronous,
     )

litellm-rust/crates/callbacks-legacy-python/src/deferred.rs

@@ -63,5 +63,151 @@ impl PendingLogging {
 }
 
 #[cfg(test)]
-#[path = "../tests/deferred.rs"]
-mod tests;
+mod tests {
+    use std::ffi::CStr;
+
+    use pyo3::prelude::*;
+    use pyo3::types::PyDict;
+    use rstest::rstest;
+
+    use super::{PendingLogging, PendingSuccess};
+    use crate::PythonLogger;
+    use crate::test_support::{local, namespace, run};
+
+    /// A deferred success for the namespace's `logger` and `response`, bound as `pending`.
+    fn defer<'py>(py: Python<'py>, script: &CStr) -> Bound<'py, PyDict> {
+        let locals = namespace(py, c"response = object()");
+        run(py, &locals, script);
+        let pending = Py::new(
+            py,
+            PendingLogging {
+                pending: Some(PendingSuccess {
+                    logger: PythonLogger::new(local(&locals, "logger").unbind()),
+                    response: Some(local(&locals, "response").unbind()),
+                    start: py.None(),
+                    end: Some(py.None()),
+                }),
+            },
+        )
+        .unwrap();
+        locals.set_item("pending", pending).unwrap();
+        locals
+    }
+
+    #[test]
+    fn release_enqueues_the_success_once_in_the_releasing_context() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = defer(
+                py,
+                c"
+from contextvars import ContextVar
+
+marker = ContextVar('marker', default='unset')
+observed = []
+
+def on_enqueue(coroutine):
+    observed.append(marker.get())
+    pending.release(True)
+
+logger.on_enqueue = on_enqueue
+",
+            );
+            run(
+                py,
+                &locals,
+                c"
+marker.set('release')
+pending.release(True)
+pending.release(True)
+assert observed == ['release'], observed
+assert logger.names() == ['async_success_handler', 'enqueued'], logger.calls
+assert logger.calls[0][1] is response
+",
+            );
+        });
+    }
+
+    #[test]
+    fn a_blocked_release_drops_the_success_for_good() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = defer(py, c"");
+            run(
+                py,
+                &locals,
+                c"
+pending.release(False)
+pending.release(True)
+assert logger.calls == [], logger.calls
+",
+            );
+        });
+    }
+
+    #[rstest]
+    #[case::ordinary_error(c"RuntimeError('queue full')", false)]
+    #[case::cancellation(c"asyncio.CancelledError()", true)]
+    fn a_failed_enqueue_closes_the_coroutine_and_is_never_replayed(
+        #[case] failure: &CStr,
+        #[case] propagates: bool,
+    ) {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = defer(
+                py,
+                c"
+import asyncio
+
+def on_enqueue(coroutine):
+    raise failure
+
+logger.on_enqueue = on_enqueue
+",
+            );
+            locals
+                .set_item("failure", py.eval(failure, None, Some(&locals)).unwrap())
+                .unwrap();
+            let released = local(&locals, "pending").call_method1("release", (true,));
+            match released {
+                Ok(_) => assert!(!propagates),
+                Err(error) => {
+                    assert!(propagates);
+                    assert!(error.value(py).is(local(&locals, "failure")));
+                }
+            }
+            locals.set_item("propagates", propagates).unwrap();
+            run(
+                py,
+                &locals,
+                c"
+pending.release(True)
+assert logger.names() == ['async_success_handler', 'enqueued', 'closed'], logger.calls
+assert unraisable_from(logger) == ([] if propagates else [failure])
+",
+            );
+        });
+    }
+
+    #[test]
+    fn an_unreleased_success_does_not_keep_its_logger_alive() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = defer(py, c"");
+            run(
+                py,
+                &locals,
+                c"
+import gc
+import weakref
+
+logger.pending = pending
+reference = weakref.ref(logger)
+del logger, pending
+gc.collect()
+assert reference() is None
+",
+            );
+        });
+    }
+}

litellm-rust/crates/callbacks-legacy-python/src/lib.rs

@@ -1,9 +1,10 @@
 //! The legacy `@client` wrapper as the native call sees it: litellm's `Logging` object, the
-//! sync and async callback registries it fans out to, the deployment hooks, the deferred
-//! proxy release, and the kwargs rewrites the wrapper makes on the way in (credential-name
-//! inheritance, budget and retry-count limits). All of it sits behind one
+//! sync and async callback registries it fans out to, the deployment hooks and the deferred
+//! proxy release. All of it sits behind one
 //! [`PythonLifecycle`](litellm_host_python::PythonLifecycle), so the driver, the routes and
-//! core never learn which Python object is on the other end.
+//! core never learn which Python object is on the other end. The SDK's own request policy
+//! (credential inheritance, the budget and retry limits) is the driver's preflight, not this
+//! crate's.
 //!
 //! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`]
 //! is where those objects live, and [`run_legacy_call`] is how a route hands them over
@@ -14,15 +15,12 @@ mod call;
 mod callbacks;
 mod deferred;
 mod logger;
-mod preparation;
 mod python;
-#[cfg(test)]
-#[path = "../tests/support.rs"]
-mod test_support;
-
 pub(crate) use adapter::LegacyLogging;
 pub use adapter::{LegacySurface, PassThroughStream};
 pub use call::{PublicCall, run_legacy_call};
 pub(crate) use callbacks::{LegacyCallbacks, is_internal_call};
 pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup};
-pub(crate) use preparation::prepare;
+
+#[cfg(test)]
+mod test_support;

litellm-rust/crates/callbacks-legacy-python/src/python.rs

@@ -19,18 +19,12 @@ pub(crate) enum LegacyPython {
     Streaming(Streaming),
 }
 
-/// The `@client` wrapper around the call: `function_setup`, limits, credentials,
-/// response metadata and the correlation context.
+/// The `@client` wrapper around the call: `function_setup`, response metadata and the
+/// correlation context.
 #[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
 pub(crate) enum Wrapper {
     #[strum(serialize = "setup")]
     Setup,
-    #[strum(serialize = "check_limits")]
-    CheckLimits,
-    #[strum(serialize = "credential_list")]
-    CredentialList,
-    #[strum(serialize = "warn_unknown_credential")]
-    WarnUnknownCredential,
     #[strum(serialize = "is_internal_call")]
     IsInternalCall,
     #[strum(serialize = "finalize")]

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyTuple};
 use crate::{LegacyLogging, LegacySurface, PublicCall};
 
 /// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
-/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
+/// `tests/unit/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
 /// signatures, and [`namespace`] binds every fake call against it.
 pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
 
@@ -50,7 +50,6 @@ FAKES = {
         logger=kwargs['logger_factory'](kwargs) if 'logger_factory' in kwargs else kwargs['logger'],
         kwargs=kwargs,
     ),
-    'check_limits': lambda arguments: arguments['logger'].check_limits(arguments),
     'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response),
     'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs(
         kwargs=kwargs,
@@ -77,8 +76,6 @@ FAKES = {
     'restore_context': lambda logger: logger.record('restore', None),
     'custom_pricing_fields': lambda: ('ocr_cost_per_page',),
     'is_internal_call': lambda: legacy.is_internal.get(),
-    'credential_list': lambda: [],
-    'warn_unknown_credential': lambda name, loaded: None,
     'before_deployment_call': lambda kwargs, call_type: kwargs['logger'].hook('pre', kwargs, call_type),
     'after_deployment_success': lambda kwargs, response, call_type: kwargs['logger'].hook(
         'success', response, call_type
@@ -135,9 +132,6 @@ class StubLogger:
         self.record(phase + '_hook', call_type)
         return self.hooks.get(phase, lambda value: 'awaitable')(value)
 
-    def check_limits(self, arguments):
-        self.record('check_limits', arguments)
-
     def failure_handler(self, error, trace, start, end):
         self.record('failure_handler', error)
 

litellm-rust/crates/callbacks-legacy-python/tests/deferred.rs

@@ -1,146 +0,0 @@
-use std::ffi::CStr;
-
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-use rstest::rstest;
-
-use super::{PendingLogging, PendingSuccess};
-use crate::PythonLogger;
-use crate::test_support::{local, namespace, run};
-
-/// A deferred success for the namespace's `logger` and `response`, bound as `pending`.
-fn defer<'py>(py: Python<'py>, script: &CStr) -> Bound<'py, PyDict> {
-    let locals = namespace(py, c"response = object()");
-    run(py, &locals, script);
-    let pending = Py::new(
-        py,
-        PendingLogging {
-            pending: Some(PendingSuccess {
-                logger: PythonLogger::new(local(&locals, "logger").unbind()),
-                response: Some(local(&locals, "response").unbind()),
-                start: py.None(),
-                end: Some(py.None()),
-            }),
-        },
-    )
-    .unwrap();
-    locals.set_item("pending", pending).unwrap();
-    locals
-}
-
-#[test]
-fn release_enqueues_the_success_once_in_the_releasing_context() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = defer(
-            py,
-            c"
-from contextvars import ContextVar
-
-marker = ContextVar('marker', default='unset')
-observed = []
-
-def on_enqueue(coroutine):
-    observed.append(marker.get())
-    pending.release(True)
-
-logger.on_enqueue = on_enqueue
-",
-        );
-        run(
-            py,
-            &locals,
-            c"
-marker.set('release')
-pending.release(True)
-pending.release(True)
-assert observed == ['release'], observed
-assert logger.names() == ['async_success_handler', 'enqueued'], logger.calls
-assert logger.calls[0][1] is response
-",
-        );
-    });
-}
-
-#[test]
-fn a_blocked_release_drops_the_success_for_good() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = defer(py, c"");
-        run(
-            py,
-            &locals,
-            c"
-pending.release(False)
-pending.release(True)
-assert logger.calls == [], logger.calls
-",
-        );
-    });
-}
-
-#[rstest]
-#[case::ordinary_error(c"RuntimeError('queue full')", false)]
-#[case::cancellation(c"asyncio.CancelledError()", true)]
-fn a_failed_enqueue_closes_the_coroutine_and_is_never_replayed(
-    #[case] failure: &CStr,
-    #[case] propagates: bool,
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = defer(
-            py,
-            c"
-import asyncio
-
-def on_enqueue(coroutine):
-    raise failure
-
-logger.on_enqueue = on_enqueue
-",
-        );
-        locals
-            .set_item("failure", py.eval(failure, None, Some(&locals)).unwrap())
-            .unwrap();
-        let released = local(&locals, "pending").call_method1("release", (true,));
-        match released {
-            Ok(_) => assert!(!propagates),
-            Err(error) => {
-                assert!(propagates);
-                assert!(error.value(py).is(local(&locals, "failure")));
-            }
-        }
-        locals.set_item("propagates", propagates).unwrap();
-        run(
-            py,
-            &locals,
-            c"
-pending.release(True)
-assert logger.names() == ['async_success_handler', 'enqueued', 'closed'], logger.calls
-assert unraisable_from(logger) == ([] if propagates else [failure])
-",
-        );
-    });
-}
-
-#[test]
-fn an_unreleased_success_does_not_keep_its_logger_alive() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = defer(py, c"");
-        run(
-            py,
-            &locals,
-            c"
-import gc
-import weakref
-
-logger.pending = pending
-reference = weakref.ref(logger)
-del logger, pending
-gc.collect()
-assert reference() is None
-",
-        );
-    });
-}

litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs

@@ -1,282 +0,0 @@
-use std::ffi::CStr;
-
-use litellm_host::event::{FailureOrigin, Timing};
-use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle};
-use pyo3::exceptions::asyncio::CancelledError;
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-use rstest::rstest;
-
-use super::LegacyLogging;
-use crate::test_support::{legacy_call, local, namespace, run};
-
-const CALL: &CStr = c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-kwargs = {'logger': logger, 'document': document}
-";
-
-const TIMING: Timing = Timing {
-    start_time: 0.0,
-    end_time: 1.0,
-};
-
-fn begin<'py>(
-    py: Python<'py>,
-    locals: &Bound<'py, PyDict>,
-    asynchronous: bool,
-) -> (LegacyLogging, LifecycleStep) {
-    let mut logging = legacy_call(py, locals, asynchronous);
-    let kwargs = local(locals, "kwargs")
-        .cast_into::<PyDict>()
-        .unwrap()
-        .unbind();
-    let step = logging.begin(py, kwargs, 0.0).unwrap();
-    (logging, step)
-}
-
-fn arguments<'py>(py: Python<'py>, step: LifecycleStep) -> Bound<'py, PyDict> {
-    let LifecycleStep::Arguments(arguments) = step else {
-        panic!("expected the prepared arguments");
-    };
-    arguments.into_bound(py)
-}
-
-fn awaits_deployment_hook(step: &LifecycleStep) -> bool {
-    matches!(step, LifecycleStep::Await(_))
-}
-
-#[rstest]
-#[case::synchronous(false)]
-#[case::asynchronous(true)]
-fn deployment_pre_call_hook_runs_only_for_asynchronous_calls(#[case] asynchronous: bool) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, CALL);
-        let (_, step) = begin(py, &locals, asynchronous);
-        assert_eq!(awaits_deployment_hook(&step), asynchronous);
-        let names: Vec<String> = local(&locals, "logger")
-            .call_method0("names")
-            .unwrap()
-            .extract()
-            .unwrap();
-        assert_eq!(names.contains(&"pre_hook".to_string()), asynchronous);
-    });
-}
-
-#[test]
-fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-replacement = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,ZWRpdGVk'}
-kwargs = {'logger': logger, 'document': document}
-replaced_kwargs = {'logger': logger, 'document': replacement, 'pages': [0]}
-",
-        );
-        let (mut logging, step) = begin(py, &locals, true);
-        assert!(awaits_deployment_hook(&step));
-        let step = logging
-            .resume(py, Ok(local(&locals, "replaced_kwargs").unbind()))
-            .unwrap();
-        locals.set_item("prepared", arguments(py, step)).unwrap();
-        run(
-            py,
-            &locals,
-            c"
-assert prepared['document'] is replacement
-assert prepared['pages'] is replaced_kwargs['pages']
-assert prepared['litellm_logging_obj'] is logger
-assert 'litellm_logging_obj' not in replaced_kwargs
-[checked] = [value for name, value in logger.calls if name == 'check_limits']
-assert checked is prepared
-",
-        );
-    });
-}
-
-#[rstest]
-#[case::synchronous(false)]
-#[case::asynchronous(true)]
-fn a_keyword_the_bridge_never_reads_reaches_every_reader_as_the_callers_object(
-    #[case] asynchronous: bool,
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-opaque = object()
-hooked = []
-logger.hooks = {'pre': lambda kwargs: hooked.append(kwargs['vendor_extension']) or kwargs}
-kwargs = {'logger': logger, 'vendor_extension': opaque}
-",
-        );
-        let (mut logging, step) = begin(py, &locals, asynchronous);
-        let step = match step {
-            LifecycleStep::Await(hook_result) => logging.resume(py, Ok(hook_result)).unwrap(),
-            step => step,
-        };
-        locals.set_item("prepared", arguments(py, step)).unwrap();
-        locals.set_item("asynchronous", asynchronous).unwrap();
-        run(
-            py,
-            &locals,
-            c"
-assert prepared['vendor_extension'] is opaque
-[checked] = [value for name, value in logger.calls if name == 'check_limits']
-assert checked['vendor_extension'] is opaque
-assert hooked == ([opaque] if asynchronous else []), hooked
-",
-        );
-    });
-}
-
-#[test]
-fn response_returned_by_the_post_call_hook_is_finalized_and_returned() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-kwargs = {'logger': logger}
-response = object()
-replacement = object()
-logger.hooks = {'pre': lambda kwargs: kwargs}
-",
-        );
-        let (mut logging, _) = begin(py, &locals, true);
-        logging
-            .resume(py, Ok(local(&locals, "kwargs").unbind()))
-            .unwrap();
-        let step = logging
-            .after_success(py, local(&locals, "response").unbind(), TIMING)
-            .unwrap();
-        assert!(awaits_deployment_hook(&step));
-        let step = logging
-            .resume(py, Ok(local(&locals, "replacement").unbind()))
-            .unwrap();
-        let LifecycleStep::Response(returned) = step else {
-            panic!("expected the finalized response");
-        };
-        assert!(returned.bind(py).is(local(&locals, "replacement")));
-        run(
-            py,
-            &locals,
-            c"
-[finalized] = [value for name, value in logger.calls if name == 'finalize']
-assert finalized is replacement
-",
-        );
-    });
-}
-
-#[rstest]
-#[case::pre_call(false)]
-#[case::post_call(true)]
-fn cancelling_a_deployment_hook_ends_the_call_with_that_cancellation(#[case] post_call: bool) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"kwargs = {'logger': logger}\nresponse = object()");
-        let (mut logging, _) = begin(py, &locals, true);
-        if post_call {
-            logging
-                .resume(py, Ok(local(&locals, "kwargs").unbind()))
-                .unwrap();
-            logging
-                .after_success(py, local(&locals, "response").unbind(), TIMING)
-                .unwrap();
-        }
-        let cancellation = CancelledError::new_err("cancelled");
-        let cancelled = cancellation.value(py).clone();
-        let error = logging.resume(py, Err(cancellation)).err().unwrap();
-        assert!(error.value(py).is(&cancelled));
-        let names: Vec<String> = local(&locals, "logger")
-            .call_method0("names")
-            .unwrap()
-            .extract()
-            .unwrap();
-        assert!(!names.iter().any(|name| name.contains("handler")));
-    });
-}
-
-#[rstest]
-#[case::hook_completed(false)]
-#[case::hook_cancelled(true)]
-fn failure_callbacks_run_after_the_failure_hook_however_it_ends(#[case] cancelled: bool) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"kwargs = {'logger': logger}\nfailure = ValueError('provider')",
-        );
-        let (mut logging, _) = begin(py, &locals, true);
-        logging
-            .resume(py, Ok(local(&locals, "kwargs").unbind()))
-            .unwrap();
-        let failure = PyErr::from_value(local(&locals, "failure"));
-        let failed = LifecycleEvent::Failed {
-            timing: TIMING,
-            origin: FailureOrigin::Call,
-            error: &failure,
-        };
-        let step = logging.emit(py, failed).unwrap();
-        assert!(awaits_deployment_hook(&step));
-        let hook_result = if cancelled {
-            Err(CancelledError::new_err("cancelled"))
-        } else {
-            Ok(py.None())
-        };
-        assert!(matches!(
-            logging.resume(py, hook_result).unwrap(),
-            LifecycleStep::Await(_)
-        ));
-        run(
-            py,
-            &locals,
-            c"
-assert logger.names()[-3:] == ['failure_hook', 'failure_handler', 'async_failure_handler'], logger.calls
-assert all(value is failure for name, value in logger.calls if name.endswith('_handler'))
-",
-        );
-    });
-}
-
-#[rstest]
-#[case::synchronous(false)]
-#[case::asynchronous(true)]
-fn a_limit_rejected_before_the_call_surfaces_as_the_callers_error(#[case] asynchronous: bool) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-class BudgetExceeded(Exception):
-    pass
-
-rejection = BudgetExceeded('over budget')
-
-class LimitedLogger(StubLogger):
-    def check_limits(self, arguments):
-        raise rejection
-
-logger = LimitedLogger()
-logger.hooks = {'pre': lambda kwargs: kwargs}
-kwargs = {'logger': logger}
-",
-        );
-        let mut logging = legacy_call(py, &locals, asynchronous);
-        let kwargs = local(&locals, "kwargs")
-            .cast_into::<PyDict>()
-            .unwrap()
-            .unbind();
-        let result = logging.begin(py, kwargs, 0.0).and_then(|step| match step {
-            LifecycleStep::Await(_) => logging.resume(py, Ok(local(&locals, "kwargs").unbind())),
-            step => Ok(step),
-        });
-        let error = result.err().unwrap();
-        assert!(error.value(py).is(local(&locals, "rejection")));
-    });
-}

litellm-rust/crates/callbacks-legacy-python/tests/payload.rs

@@ -1,523 +0,0 @@
-use std::ffi::CStr;
-
-use litellm_auth::SecretValue;
-use litellm_host::event::{MachineEvent, RawResponse, RequestContext, WireRequest};
-use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, to_py};
-use proptest::prelude::*;
-use pyo3::prelude::*;
-use rstest::rstest;
-use serde_json::{Map, Value, json};
-
-use super::LegacyLogging;
-use crate::PythonLogger;
-use crate::test_support::{legacy_call, local, namespace, run};
-
-/// The payload phases of `Logging` on top of `StubLogger`, with `pre_call` handing the
-/// payload to the case's `on_pre_call`.
-const PAYLOAD_LOGGER: &CStr = c"
-class Request:
-    pass
-
-class PayloadLogger(StubLogger):
-    def update_from_kwargs(self, **update):
-        self.update = update
-
-    def pre_call(self, input, api_key, additional_args):
-        self.record('pre_call', None)
-        self.pre = additional_args
-        self.pre_api_key = api_key
-        on_pre_call(additional_args)
-
-    def post_call(self, original_response, api_key, additional_args):
-        self.record('post_call', None)
-        self.post = (original_response, api_key, additional_args)
-
-request = Request()
-kwargs = {}
-logger = PayloadLogger()
-on_pre_call = lambda additional_args: None
-check = lambda: None
-";
-
-const DOCUMENT: &str = "data:application/pdf;base64,YWJj";
-const EDITED: &str = "data:application/pdf;base64,ZWRpdGVk";
-
-fn document(source: &str) -> Value {
-    json!({"type": "document_url", "document_url": source})
-}
-
-fn before_send(script: &CStr, body: Value) -> WireRequest {
-    before_send_with_secrets(script, json!({}), body, &[])
-}
-
-/// Runs `before_send` over `body` for a route whose parameters are `optional_params`, with
-/// the Python objects `script` binds, then delivers the provider's raw response the way the
-/// driver does and runs the script's `check()`.
-fn before_send_with_secrets(
-    script: &CStr,
-    optional_params: Value,
-    body: Value,
-    secret_fields: &[&str],
-) -> WireRequest {
-    before_send_bound(&[], script, optional_params, body, secret_fields)
-}
-
-/// [`before_send_with_secrets`] with `bindings` placed in the namespace before `script` runs.
-fn before_send_bound(
-    bindings: &[(&str, &Value)],
-    script: &CStr,
-    optional_params: Value,
-    body: Value,
-    secret_fields: &[&str],
-) -> WireRequest {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, PAYLOAD_LOGGER);
-        for &(name, value) in bindings {
-            locals.set_item(name, to_py(py, value).unwrap()).unwrap();
-        }
-        run(py, &locals, script);
-        let mut logging = LegacyLogging {
-            logger: Some(PythonLogger::new(local(&locals, "logger").unbind())),
-            ..legacy_call(py, &locals, false)
-        };
-        let context = RequestContext {
-            model: "model".into(),
-            custom_llm_provider: "provider".into(),
-            optional_params,
-            secret_fields: secret_fields.iter().map(|name| name.to_string()).collect(),
-            api_key: Some(SecretValue::new("route-key")),
-        };
-        let wire = WireRequest {
-            url: "https://provider.invalid/ocr".into(),
-            headers: vec![("x-route".into(), "route".into())],
-            body,
-        };
-        let step = logging.before_send(py, Box::new(wire), &context).unwrap();
-        let raw = MachineEvent::ResponseReceived {
-            raw: RawResponse {
-                body: "raw response".into(),
-            },
-        };
-        assert!(matches!(
-            logging.emit(py, LifecycleEvent::Machine(&raw)).unwrap(),
-            LifecycleStep::Done
-        ));
-        run(py, &locals, c"check()");
-        let LifecycleStep::Wire(wire) = step else {
-            panic!("before_send did not hand back the wire request");
-        };
-        *wire
-    })
-}
-
-#[rstest]
-#[case::caller_keyword(c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-pages = [0]
-kwargs = {'document': document, 'pages': pages}
-observed = []
-on_pre_call = lambda args: observed.append(
-    (args['complete_input_dict']['document'] is document, args['complete_input_dict']['pages'] is pages)
-)
-def check():
-    assert observed == [(True, True)], observed
-")]
-#[case::request_attribute_behind_an_omitted_keyword(c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-pages = [0]
-request.document = document
-kwargs = {'pages': pages}
-observed = []
-on_pre_call = lambda args: observed.append(
-    (args['complete_input_dict']['document'] is document, args['complete_input_dict']['pages'] is pages)
-)
-def check():
-    assert observed == [(True, True)], observed
-")]
-fn passthrough_keys_reach_pre_call_as_the_callers_own_objects(#[case] script: &CStr) {
-    let body = json!({"model": "model", "document": document(DOCUMENT), "pages": [0]});
-    let wire = before_send(script, body.clone());
-    assert_eq!(wire.body, body);
-}
-
-#[test]
-fn pre_call_edit_of_a_passthrough_object_reaches_the_caller_and_the_wire() {
-    let wire = before_send(
-        c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-kwargs = {'document': document}
-def on_pre_call(args):
-    args['complete_input_dict']['document']['document_url'] = 'data:application/pdf;base64,ZWRpdGVk'
-def check():
-    assert document['document_url'] == 'data:application/pdf;base64,ZWRpdGVk'
-",
-        json!({"document": document(DOCUMENT)}),
-    );
-    assert_eq!(wire.body["document"], document(EDITED));
-}
-
-#[test]
-fn a_body_key_the_route_rewrote_is_not_the_callers_object() {
-    let wire = before_send(
-        c"
-document = {'type': 'document_url', 'document_url': 'https://example.invalid/scan.pdf'}
-kwargs = {'document': document}
-observed = []
-def on_pre_call(args):
-    observed.append(args['complete_input_dict']['document'] is document)
-    args['complete_input_dict']['document']['document_name'] = 'edited.pdf'
-def check():
-    assert observed == [False], observed
-    assert document == {'type': 'document_url', 'document_url': 'https://example.invalid/scan.pdf'}
-",
-        json!({"document": document(DOCUMENT)}),
-    );
-    assert_eq!(
-        wire.body["document"],
-        json!({"type": "document_url", "document_url": DOCUMENT, "document_name": "edited.pdf"})
-    );
-}
-
-#[test]
-fn a_caller_value_with_no_json_form_is_left_out_of_realiasing() {
-    let body = json!({"pages": [0]});
-    let wire = before_send(
-        c"
-opaque = object()
-kwargs = {'pages': opaque}
-observed = []
-on_pre_call = lambda args: observed.append(args['complete_input_dict']['pages'])
-def check():
-    assert observed == [[0]], observed
-",
-        body.clone(),
-    );
-    assert_eq!(wire.body, body);
-}
-
-#[rstest]
-#[case::body(
-    c"
-def on_pre_call(args):
-    args['complete_input_dict'] = {'replacement': True}
-"
-)]
-#[case::headers(
-    c"
-def on_pre_call(args):
-    args['headers'] = {'x-replacement': 'yes'}
-"
-)]
-fn rebinding_the_payload_envelope_does_not_reach_the_wire(#[case] script: &CStr) {
-    let body = json!({"document": document(DOCUMENT)});
-    let wire = before_send(script, body.clone());
-    assert_eq!(wire.body, body);
-    assert_eq!(wire.headers, [("x-route".to_string(), "route".to_string())]);
-}
-
-#[test]
-fn pre_call_header_edit_reaches_the_wire() {
-    let wire = before_send(
-        c"
-def on_pre_call(args):
-    args['headers']['x-callback'] = 'edited'
-",
-        json!({}),
-    );
-    assert_eq!(
-        wire.headers,
-        [
-            ("x-route".to_string(), "route".to_string()),
-            ("x-callback".to_string(), "edited".to_string()),
-        ]
-    );
-}
-
-#[test]
-fn pre_call_receives_the_wire_request_and_the_logger_its_redacted_request() {
-    let body = json!({"model": "model", "document": document(DOCUMENT)});
-    before_send_with_secrets(
-        c"
-logger_fn = lambda *args: None
-kwargs = {
-    'litellm_call_id': 'call-1',
-    'client_secret': 'shh',
-    'proxy_server_request': {'body': {}},
-    'logger_fn': logger_fn,
-    'litellm_request_debug': True,
-    'ocr_cost_per_page': 0.05,
-}
-observed = []
-on_pre_call = observed.append
-def check():
-    [args] = observed
-    assert args['api_base'] == 'https://provider.invalid/ocr', args
-    assert args['complete_input_dict'] == {
-        'model': 'model',
-        'document': {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'},
-    }, args
-    update = logger.update
-    assert update['model'] == 'model' and update['custom_llm_provider'] == 'provider', update
-    assert update['litellm_params']['litellm_call_id'] == 'call-1', update
-    assert update['litellm_params']['api_base'] == 'https://provider.invalid/ocr', update
-    assert update['litellm_params']['logger_fn'] is logger_fn, update
-    assert update['litellm_params']['litellm_request_debug'] is True, update
-    assert update['litellm_params']['ocr_cost_per_page'] == 0.05, update
-    assert update['kwargs']['client_secret'] == '****', update
-    assert 'proxy_server_request' not in update['kwargs'], update
-    assert update['optional_params']['client_secret'] == '****', update
-",
-        json!({"client_secret": "shh"}),
-        body,
-        &["client_secret"],
-    );
-}
-
-#[rstest]
-#[case::added_key(
-    c"
-def on_pre_call(args):
-    args['complete_input_dict']['include_image_base64'] = True
-",
-    json!({"document": document(DOCUMENT), "include_image_base64": true})
-)]
-#[case::replaced_document(
-    c"
-document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
-kwargs = {'document': document}
-def on_pre_call(args):
-    args['complete_input_dict']['document'] = {
-        'type': 'document_url', 'document_url': 'data:application/pdf;base64,ZWRpdGVk'
-    }
-def check():
-    assert document['document_url'] == 'data:application/pdf;base64,YWJj', document
-",
-    json!({"document": document(EDITED)})
-)]
-#[case::retained_body_edited_after_rebinding(
-    c"
-def on_pre_call(args):
-    retained = args['complete_input_dict']
-    args['complete_input_dict'] = {'rebound': True}
-    retained['include_image_base64'] = True
-",
-    json!({"document": document(DOCUMENT), "include_image_base64": true})
-)]
-fn pre_call_body_edits_reach_the_wire(#[case] script: &CStr, #[case] expected: Value) {
-    let body = json!({"document": document(DOCUMENT)});
-    let wire = before_send(script, body);
-    assert_eq!(wire.body, expected);
-}
-
-#[test]
-fn retained_headers_edited_after_rebinding_reach_the_wire() {
-    let wire = before_send(
-        c"
-def on_pre_call(args):
-    retained = args['headers']
-    args['headers'] = {'x-rebound': 'rebound'}
-    retained['x-retained'] = 'sent'
-",
-        json!({}),
-    );
-    assert_eq!(
-        wire.headers,
-        [
-            ("x-route".to_string(), "route".to_string()),
-            ("x-retained".to_string(), "sent".to_string()),
-        ]
-    );
-}
-
-#[test]
-fn post_call_receives_the_raw_response_the_route_key_and_the_body_and_headers_pre_call_saw() {
-    before_send(
-        c"
-def check():
-    original_response, api_key, additional_args = logger.post
-    assert original_response == 'raw response', original_response
-    assert api_key == logger.pre_api_key == 'route-key', (api_key, logger.pre_api_key)
-    assert additional_args == {
-        'complete_input_dict': logger.pre['complete_input_dict'],
-        'headers': logger.pre['headers'],
-    }, additional_args
-    assert additional_args['complete_input_dict'] is logger.pre['complete_input_dict']
-    assert additional_args['headers'] is logger.pre['headers']
-",
-        json!({"document": document(DOCUMENT)}),
-    );
-}
-
-#[test]
-fn every_request_runs_the_full_pre_call_and_post_call() {
-    let wire = before_send(
-        c"
-def on_pre_call(args):
-    args['complete_input_dict']['include_image_base64'] = True
-def check():
-    assert logger.names() == ['pre_call', 'post_call'], logger.calls
-",
-        json!({"document": document(DOCUMENT)}),
-    );
-    assert_eq!(
-        wire.body,
-        json!({"document": document(DOCUMENT), "include_image_base64": true})
-    );
-}
-
-/// What one pre-call callback does to the payload it is handed.
-#[derive(Clone, Debug)]
-enum Edit {
-    Nothing,
-    Set(String, Value),
-    Remove(String),
-    Rebind(Value),
-    RebindThenSetRetained(String, Value),
-}
-
-impl Edit {
-    fn script(&self) -> Value {
-        match self {
-            Self::Nothing => json!({"kind": "nothing"}),
-            Self::Set(key, value) => json!({"kind": "set", "key": key, "value": value}),
-            Self::Remove(key) => json!({"kind": "remove", "key": key}),
-            Self::Rebind(value) => json!({"kind": "rebind", "value": value}),
-            Self::RebindThenSetRetained(key, value) => {
-                json!({"kind": "rebind_then_set_retained", "key": key, "value": value})
-            }
-        }
-    }
-
-    /// The legacy contract: the provider is sent the body object `pre_call` received, as
-    /// the callback left it. Rebinding the envelope's key points the envelope elsewhere and
-    /// leaves that object alone.
-    fn sent(&self, body: &Map<String, Value>) -> Value {
-        let mut sent = body.clone();
-        match self {
-            Self::Nothing | Self::Rebind(_) => {}
-            Self::Set(key, value) | Self::RebindThenSetRetained(key, value) => {
-                sent.insert(key.clone(), value.clone());
-            }
-            Self::Remove(key) => {
-                sent.remove(key);
-            }
-        }
-        Value::Object(sent)
-    }
-}
-
-/// How the caller's keyword for a body key relates to what the route sends under it.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum Caller {
-    PassedUnchanged,
-    RewrittenByTheRoute,
-    NotPassed,
-}
-
-const MODEL: &CStr = c"
-aliased = {}
-def on_pre_call(args):
-    body = args['complete_input_dict']
-    aliased.update({name: body[name] is kwargs[name] for name in unchanged})
-    kind = edit['kind']
-    if kind == 'set':
-        body[edit['key']] = edit['value']
-    elif kind == 'remove':
-        body.pop(edit['key'], None)
-    elif kind == 'rebind':
-        args['complete_input_dict'] = edit['value']
-    elif kind == 'rebind_then_set_retained':
-        args['complete_input_dict'] = {}
-        body[edit['key']] = edit['value']
-def check():
-    assert aliased == {name: True for name in unchanged}, aliased
-    assert logger.names() == ['pre_call', 'post_call'], logger.calls
-";
-
-fn json_value() -> impl Strategy<Value = Value> {
-    let leaf = prop_oneof![
-        Just(Value::Null),
-        any::<bool>().prop_map(Value::from),
-        any::<i64>().prop_map(Value::from),
-        any::<f64>()
-            .prop_filter("JSON has no NaN or infinity", |number| number.is_finite())
-            .prop_map(Value::from),
-        ".{0,8}".prop_map(Value::from),
-    ];
-    leaf.prop_recursive(3, 24, 4, |inner| {
-        prop_oneof![
-            prop::collection::vec(inner.clone(), 0..4).prop_map(Value::from),
-            prop::collection::btree_map(key(), inner, 0..4)
-                .prop_map(|fields| Value::Object(fields.into_iter().collect())),
-        ]
-    })
-}
-
-fn key() -> impl Strategy<Value = String> {
-    "[a-z]{1,6}"
-}
-
-fn caller() -> impl Strategy<Value = Caller> {
-    prop_oneof![
-        Just(Caller::PassedUnchanged),
-        Just(Caller::RewrittenByTheRoute),
-        Just(Caller::NotPassed),
-    ]
-}
-
-fn edit() -> impl Strategy<Value = Edit> {
-    prop_oneof![
-        Just(Edit::Nothing),
-        (key(), json_value()).prop_map(|(key, value)| Edit::Set(key, value)),
-        key().prop_map(Edit::Remove),
-        json_value().prop_map(Edit::Rebind),
-        (key(), json_value()).prop_map(|(key, value)| Edit::RebindThenSetRetained(key, value)),
-    ]
-}
-
-proptest! {
-    #![proptest_config(ProptestConfig::with_cases(128))]
-
-    /// For any body, any caller keywords and any callback edit: every keyword the route
-    /// sends unchanged reaches `pre_call` as the caller's own object, and the provider is
-    /// sent exactly what the model says, so a callback that edits nothing changes nothing.
-    #[test]
-    fn the_wire_is_the_body_pre_call_received_as_the_callback_left_it(
-        fields in prop::collection::btree_map(key(), (json_value(), caller()), 0..5),
-        edit in edit(),
-    ) {
-        let body: Map<String, Value> = fields
-            .iter()
-            .map(|(name, (value, _))| (name.clone(), value.clone()))
-            .collect();
-        let kwargs: Map<String, Value> = fields
-            .iter()
-            .filter_map(|(name, (value, caller))| match caller {
-                Caller::PassedUnchanged => Some((name.clone(), value.clone())),
-                Caller::RewrittenByTheRoute => Some((name.clone(), json!([value]))),
-                Caller::NotPassed => None,
-            })
-            .collect();
-        let unchanged: Value = fields
-            .iter()
-            .filter(|(_, (_, caller))| *caller == Caller::PassedUnchanged)
-            .map(|(name, _)| Value::from(name.clone()))
-            .collect();
-
-        let wire = before_send_bound(
-            &[
-                ("kwargs", &Value::Object(kwargs)),
-                ("unchanged", &unchanged),
-                ("edit", &edit.script()),
-            ],
-            MODEL,
-            json!({}),
-            Value::Object(body.clone()),
-            &[],
-        );
-
-        prop_assert_eq!(wire.body, edit.sent(&body));
-        prop_assert_eq!(wire.headers, [("x-route".to_string(), "route".to_string())]);
-    }
-}

litellm-rust/crates/callbacks-legacy-python/tests/terminal.rs

@@ -1,291 +0,0 @@
-use std::ffi::CStr;
-
-use litellm_host::event::{FailureOrigin, Timing};
-use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle};
-use pyo3::exceptions::PyRuntimeError;
-use pyo3::exceptions::asyncio::CancelledError;
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-use rstest::rstest;
-
-use super::LegacyLogging;
-use crate::PythonLogger;
-use crate::test_support::{legacy_call, local, namespace, run};
-
-const TIMING: Timing = Timing {
-    start_time: 0.0,
-    end_time: 1.0,
-};
-
-fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyLogging {
-    LegacyLogging {
-        logger: Some(PythonLogger::new(local(locals, "logger").unbind())),
-        ..legacy_call(py, locals, asynchronous)
-    }
-}
-
-fn succeed(
-    py: Python<'_>,
-    locals: &Bound<'_, PyDict>,
-    logging: &mut LegacyLogging,
-) -> LifecycleStep {
-    let response = local(locals, "response").unbind();
-    logging
-        .emit(
-            py,
-            LifecycleEvent::Succeeded {
-                timing: TIMING,
-                response: &response,
-            },
-        )
-        .unwrap()
-}
-
-fn fail(py: Python<'_>, locals: &Bound<'_, PyDict>, logging: &mut LegacyLogging) -> LifecycleStep {
-    let failure = PyErr::from_value(local(locals, "failure"));
-    logging
-        .emit(
-            py,
-            LifecycleEvent::Failed {
-                timing: TIMING,
-                origin: FailureOrigin::Host,
-                error: &failure,
-            },
-        )
-        .unwrap()
-}
-
-#[rstest]
-#[case::sync_listened(false, c"", &["submit"])]
-#[case::async_listened(
-    true,
-    c"",
-    &["async_success_handler", "enqueued", "sync_success_for_async_call"]
-)]
-#[case::async_deferred(true, c"logger._defer_async_logging = True", &["sync_success_for_async_call"])]
-#[case::async_with_fallbacks(true, c"kwargs = {'fallbacks': ['other']}", &["sync_success_for_async_call"])]
-fn success_reaches_the_logging_handlers(
-    #[case] asynchronous: bool,
-    #[case] script: &CStr,
-    #[case] expected: &[&str],
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"response = object()");
-        run(py, &locals, script);
-        let mut logging = logged(py, &locals, asynchronous);
-        assert!(matches!(
-            succeed(py, &locals, &mut logging),
-            LifecycleStep::Done
-        ));
-        let names: Vec<String> = local(&locals, "logger")
-            .call_method0("names")
-            .unwrap()
-            .extract()
-            .unwrap();
-        assert_eq!(names, expected);
-        run(
-            py,
-            &locals,
-            c"
-assert all(value is response for name, value in logger.calls if name.endswith('_handler'))
-assert hasattr(logger, '_native_pending_logging') == getattr(logger, '_defer_async_logging', False)
-",
-        );
-    });
-}
-
-#[rstest]
-#[case::synchronous(false, &["failure_handler"])]
-#[case::asynchronous(true, &[])]
-fn internal_calls_skip_failure_callbacks_only_when_asynchronous(
-    #[case] asynchronous: bool,
-    #[case] expected: &[&str],
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"failure = ValueError('provider')");
-        let mut logging = LegacyLogging {
-            internal: true,
-            ..logged(py, &locals, asynchronous)
-        };
-        assert!(matches!(
-            fail(py, &locals, &mut logging),
-            LifecycleStep::Done
-        ));
-        let names: Vec<String> = local(&locals, "logger")
-            .call_method0("names")
-            .unwrap()
-            .extract()
-            .unwrap();
-        assert_eq!(names, expected);
-    });
-}
-
-#[test]
-fn internal_async_calls_skip_the_async_success_fan_out() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"response = object()");
-        let mut logging = LegacyLogging {
-            internal: true,
-            ..logged(py, &locals, true)
-        };
-        succeed(py, &locals, &mut logging);
-        run(
-            py,
-            &locals,
-            c"assert logger.names() == ['sync_success_for_async_call'], logger.calls",
-        );
-    });
-}
-
-#[test]
-fn a_failing_success_callback_is_reported_without_replacing_the_response() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-response = object()
-failure = ValueError('terminal diagnostic')
-
-class FailingLogger(StubLogger):
-    def handle_sync_success_callbacks_for_async_calls(self, *args):
-        raise failure
-
-logger = FailingLogger()
-",
-        );
-        let mut logging = logged(py, &locals, true);
-        assert!(matches!(
-            succeed(py, &locals, &mut logging),
-            LifecycleStep::Done
-        ));
-        assert!(
-            logging
-                .response
-                .as_ref()
-                .unwrap()
-                .bind(py)
-                .is(local(&locals, "response"))
-        );
-        run(py, &locals, c"assert unraisable_from(logger) == [failure]");
-    });
-}
-
-#[rstest]
-#[case::sync_listened(false, c"", &["failure_handler"])]
-#[case::async_listened(true, c"", &["failure_handler", "async_failure_handler"])]
-fn failure_reaches_the_logging_handlers(
-    #[case] asynchronous: bool,
-    #[case] script: &CStr,
-    #[case] expected: &[&str],
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"failure = ValueError('provider')");
-        run(py, &locals, script);
-        let mut logging = logged(py, &locals, asynchronous);
-        let step = fail(py, &locals, &mut logging);
-        let awaits_async_handler = expected.contains(&"async_failure_handler");
-        assert_eq!(
-            matches!(step, LifecycleStep::Await(_)),
-            awaits_async_handler
-        );
-        let names: Vec<String> = local(&locals, "logger")
-            .call_method0("names")
-            .unwrap()
-            .extract()
-            .unwrap();
-        assert_eq!(names, expected);
-        run(
-            py,
-            &locals,
-            c"assert all(value is failure for name, value in logger.calls if name.endswith('_handler'))",
-        );
-    });
-}
-
-#[test]
-fn a_failing_sync_failure_callback_keeps_the_error_and_still_runs_the_async_family() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(
-            py,
-            c"
-failure = ValueError('selected')
-
-class FailingLogger(StubLogger):
-    def failure_handler(self, error, trace, start, end):
-        self.record('failure_handler', error)
-        raise RuntimeError('handler failed')
-
-logger = FailingLogger()
-",
-        );
-        let mut logging = logged(py, &locals, true);
-        assert!(matches!(
-            fail(py, &locals, &mut logging),
-            LifecycleStep::Await(_)
-        ));
-        assert!(
-            logging
-                .error
-                .as_ref()
-                .unwrap()
-                .bind(py)
-                .is(local(&locals, "failure"))
-        );
-        run(
-            py,
-            &locals,
-            c"assert logger.names() == ['failure_handler', 'async_failure_handler'], logger.calls",
-        );
-    });
-}
-
-#[rstest]
-#[case::completed(None, true)]
-#[case::handler_error(Some(false), true)]
-#[case::cancelled(Some(true), false)]
-fn the_async_failure_handler_ends_the_call_unless_it_was_cancelled(
-    #[case] error: Option<bool>,
-    #[case] done: bool,
-) {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"failure = ValueError('provider')");
-        let mut logging = logged(py, &locals, true);
-        fail(py, &locals, &mut logging);
-        let result = match error {
-            None => Ok(py.None()),
-            Some(false) => Err(PyRuntimeError::new_err("handler failed")),
-            Some(true) => Err(CancelledError::new_err("cancelled")),
-        };
-        let expected = result.as_ref().err().map(|error| error.value(py).clone());
-        match logging.resume(py, result) {
-            Ok(step) => assert!(done && matches!(step, LifecycleStep::Done)),
-            Err(propagated) => {
-                assert!(!done);
-                assert!(propagated.value(py).is(expected.unwrap()));
-            }
-        }
-    });
-}
-
-#[test]
-fn closing_restores_the_correlation_context_once() {
-    Python::initialize();
-    Python::attach(|py| {
-        let locals = namespace(py, c"");
-        let mut logging = logged(py, &locals, true);
-        logging.close(py);
-        logging.close(py);
-        run(
-            py,
-            &locals,
-            c"assert logger.names() == ['restore'], logger.calls",
-        );
-    });
-}

litellm-rust/crates/config/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "litellm-config"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-types.workspace = true
+serde.workspace = true
+serde_yaml_ng = "0.10.0"
+thiserror.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tempfile.workspace = true

litellm-rust/crates/config/src/error.rs

@@ -0,0 +1,7 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("could not read config")]
+    Read(#[from] std::io::Error),
+    #[error("invalid YAML config")]
+    Parse(#[from] serde_yaml_ng::Error),
+}

litellm-rust/crates/config/src/lib.rs

@@ -0,0 +1,48 @@
+mod error;
+
+use std::path::Path;
+
+use litellm_auth_types::SecretValue;
+use serde::Deserialize;
+
+pub use error::Error;
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Config {
+    pub model_list: Box<[Model]>,
+    #[serde(default)]
+    pub general_settings: GeneralSettings,
+}
+
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GeneralSettings {
+    pub master_key: Option<SecretValue>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Model {
+    pub model_name: String,
+    pub litellm_params: LiteLlmParams,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct LiteLlmParams {
+    pub model: String,
+    pub api_key: Option<SecretValue>,
+    pub api_base: Option<String>,
+    pub custom_llm_provider: Option<String>,
+}
+
+impl Config {
+    pub fn from_yaml(yaml: &str) -> Result<Self, Error> {
+        Ok(serde_yaml_ng::from_str(yaml)?)
+    }
+
+    pub fn load(path: impl AsRef<Path>) -> Result<Self, Error> {
+        Self::from_yaml(&std::fs::read_to_string(path)?)
+    }
+}

litellm-rust/crates/config/tests/config.rs

@@ -0,0 +1,119 @@
+use litellm_config::{Config, Error};
+use rstest::{fixture, rstest};
+use tempfile::TempDir;
+
+#[fixture]
+fn directory() -> TempDir {
+    tempfile::tempdir().unwrap()
+}
+
+#[fixture]
+fn model_list_yaml() -> &'static str {
+    r#"
+model_list:
+  - model_name: assistant
+    litellm_params:
+      model: anthropic/test-model
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: local
+    litellm_params:
+      model: test-model
+      api_base: http://localhost:8000/v1
+      custom_llm_provider: openai
+"#
+}
+
+#[rstest]
+fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) {
+    let path = directory.path().join("config.yaml");
+    std::fs::write(&path, model_list_yaml).unwrap();
+
+    let config = Config::load(path).unwrap();
+    assert_eq!(config.model_list.len(), 2);
+    let anthropic = &config.model_list[0];
+    assert_eq!(anthropic.model_name, "assistant");
+    assert_eq!(anthropic.litellm_params.model, "anthropic/test-model");
+    assert_eq!(
+        anthropic.litellm_params.api_key.as_ref().unwrap().expose(),
+        "os.environ/ANTHROPIC_API_KEY"
+    );
+    assert!(anthropic.litellm_params.api_base.is_none());
+    assert!(anthropic.litellm_params.custom_llm_provider.is_none());
+    let local = &config.model_list[1];
+    assert_eq!(local.model_name, "local");
+    assert_eq!(local.litellm_params.model, "test-model");
+    assert!(local.litellm_params.api_key.is_none());
+    assert_eq!(
+        local.litellm_params.api_base.as_deref(),
+        Some("http://localhost:8000/v1")
+    );
+    assert_eq!(
+        local.litellm_params.custom_llm_provider.as_deref(),
+        Some("openai")
+    );
+}
+
+#[rstest]
+fn config_debug_redacts_api_keys() {
+    let config = Config::from_yaml(
+        "model_list: [{model_name: assistant, litellm_params: {model: anthropic/test-model, api_key: secret-value}}]",
+    )
+    .unwrap();
+    assert_eq!(
+        config.model_list[0]
+            .litellm_params
+            .api_key
+            .as_ref()
+            .unwrap()
+            .expose(),
+        "secret-value"
+    );
+    assert!(!format!("{config:?}").contains("secret-value"));
+}
+
+#[rstest]
+#[case::malformed_yaml("model_list: [")]
+#[case::missing_model_list("{}")]
+#[case::missing_params("model_list: [{model_name: assistant}]")]
+#[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
+#[case::unsupported_settings("model_list: []\ngeneral_settings: {unknown: true}")]
+#[case::misspelled_param(
+    "model_list: [{model_name: assistant, litellm_params: {model: test, api_bsae: url}}]"
+)]
+fn rejects_malformed_incomplete_and_unsupported_config(#[case] yaml: &str) {
+    assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_))));
+}
+
+#[rstest]
+fn distinguishes_read_errors_from_parse_errors(directory: TempDir) {
+    assert!(matches!(
+        Config::load(directory.path().join("missing.yaml")),
+        Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound
+    ));
+}
+
+#[rstest]
+#[case::literal("secret-master-key")]
+#[case::reference("os.environ/LITELLM_MASTER_KEY")]
+fn loads_and_redacts_the_master_key(#[case] key: &str) {
+    let config = Config::from_yaml(&format!(
+        "model_list: []\ngeneral_settings:\n  master_key: {key}\n"
+    ))
+    .unwrap();
+    assert_eq!(
+        config
+            .general_settings
+            .master_key
+            .as_ref()
+            .unwrap()
+            .expose(),
+        key
+    );
+    assert!(!format!("{config:?}").contains(key));
+}
+
+#[rstest]
+fn missing_general_settings_has_no_master_key() {
+    let config = Config::from_yaml("model_list: []").unwrap();
+    assert!(config.general_settings.master_key.is_none());
+}

litellm-rust/crates/core-utils/Cargo.toml

@@ -13,6 +13,7 @@ serde.workspace = true
 serde_json.workspace = true
 serde_path_to_error = "0.1"
 serde_with.workspace = true
+strum.workspace = true
 thiserror.workspace = true
 url.workspace = true
 

litellm-rust/crates/core-utils/src/dot_notation_indexing.rs

@@ -0,0 +1,274 @@
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+enum Segment {
+    Field(String),
+    Every,
+    Index(usize),
+}
+
+fn parse_segments(path: &str) -> Option<Vec<Segment>> {
+    let mut segments = Vec::new();
+    let mut rest = path;
+    while !rest.is_empty() {
+        if let Some(after_open) = rest.strip_prefix('[') {
+            let (inside, after) = after_open.split_once(']')?;
+            segments.push(match inside {
+                "*" => Segment::Every,
+                index => Segment::Index(index.trim().parse().ok()?),
+            });
+            rest = after.strip_prefix('.').unwrap_or(after);
+            continue;
+        }
+        let end = rest.find(['.', '[']).unwrap_or(rest.len());
+        let (field, after) = rest.split_at(end);
+        if !field.is_empty() {
+            segments.push(Segment::Field(field.to_string()));
+        }
+        rest = after.strip_prefix('.').unwrap_or(after);
+    }
+    Some(segments)
+}
+
+fn without_path(value: Value, segments: &[Segment]) -> Value {
+    let Some((segment, tail)) = segments.split_first() else {
+        return value;
+    };
+    match (segment, value) {
+        (Segment::Field(name), Value::Object(object)) => Value::Object(
+            object
+                .into_iter()
+                .filter_map(|(key, item)| {
+                    if key != *name {
+                        return Some((key, item));
+                    }
+                    (!tail.is_empty()).then(|| (key, without_path(item, tail)))
+                })
+                .collect(),
+        ),
+        (Segment::Every, Value::Array(items)) => Value::Array(
+            items
+                .into_iter()
+                .map(|item| without_path(item, tail))
+                .collect(),
+        ),
+        (Segment::Index(index), Value::Array(items)) => Value::Array(
+            items
+                .into_iter()
+                .enumerate()
+                .map(|(position, item)| {
+                    if position == *index {
+                        without_path(item, tail)
+                    } else {
+                        item
+                    }
+                })
+                .collect(),
+        ),
+        (_, value) => value,
+    }
+}
+
+pub fn delete_nested_value(value: Value, path: &str) -> Value {
+    match parse_segments(path) {
+        Some(segments) => without_path(value, &segments),
+        None => value,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::{fixture, rstest};
+    use serde_json::json;
+
+    use super::*;
+
+    #[fixture]
+    fn body() -> Value {
+        json!({
+            "tools": [
+                {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+                {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+            ],
+            "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+            "top": 0.7
+        })
+    }
+
+    #[rstest]
+    #[case::top_level_field("top", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}
+    }))]
+    #[case::whole_object("meta", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "top": 0.7
+    }))]
+    #[case::nested_field("meta.inner.drop", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::trailing_dot("meta.inner.drop.", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::leading_and_doubled_dots(".meta..inner.drop", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::field_in_every_element("tools[*].examples", json!({
+        "tools": [
+            {"name": "t0", "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::whole_array_field_in_every_element("tools[*].arr", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"]},
+            {"name": "t1", "examples": ["b"]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::field_in_indexed_element("tools[1].examples", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::padded_index("tools[ 1 ].examples", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::field_right_after_bracket("tools[0]examples", json!({
+        "tools": [
+            {"name": "t0", "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::index_then_wildcard("tools[0].arr[*].f", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"k": 1}, {"k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::wildcard_then_index_only_where_it_exists("tools[*].arr[1].f", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    #[case::nested_wildcards("tools[*].arr[*].f", json!({
+        "tools": [
+            {"name": "t0", "examples": ["a"], "arr": [{"k": 1}, {"k": 2}]},
+            {"name": "t1", "examples": ["b"], "arr": [{"k": 3}]}
+        ],
+        "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}},
+        "top": 0.7
+    }))]
+    fn deletes_the_addressed_field(body: Value, #[case] path: &str, #[case] expected: Value) {
+        assert_eq!(delete_nested_value(body, path), expected);
+    }
+
+    #[rstest]
+    #[case::empty_path("")]
+    #[case::missing_field("missing")]
+    #[case::missing_parent("missing.field")]
+    #[case::field_through_a_scalar("top.value")]
+    #[case::field_on_an_array("tools.name")]
+    #[case::index_on_an_object("meta[0].user")]
+    #[case::wildcard_on_an_object("meta[*].user")]
+    #[case::wildcard_over_scalars("tools[*].examples[*].name")]
+    #[case::index_out_of_range("tools[5].name")]
+    #[case::every_element_itself("tools[*]")]
+    #[case::indexed_element_itself("tools[0]")]
+    #[case::nested_element_itself("tools[*].arr[0]")]
+    #[case::negative_index("tools[-1].name")]
+    #[case::non_numeric_index("tools[x].name")]
+    #[case::empty_index("tools[].name")]
+    #[case::unclosed_bracket("top[0")]
+    fn leaves_the_value_untouched(body: Value, #[case] path: &str) {
+        assert_eq!(delete_nested_value(body.clone(), path), body);
+    }
+
+    #[rstest]
+    #[case::wildcards_indices_and_nesting(
+        json!({"tools": [
+            {"name": "t0", "configs": [{"id": "c0", "remove_me": 1, "keep": 1}, {"id": "c1", "remove_me": 2, "keep": 2}], "metadata": {"drop_this": 1, "preserve": 1}},
+            {"name": "t1", "configs": [{"id": "c0", "remove_me": 3, "keep": 3}, {"id": "c1", "remove_me": 4, "keep": 4}], "metadata": {"drop_this": 2, "preserve": 2}},
+            {"name": "t2", "configs": [{"id": "c0", "remove_me": 5, "keep": 5}], "metadata": {"drop_this": 3, "preserve": 3}}
+        ]}),
+        &["tools[*].configs[1].remove_me", "tools[1].metadata.drop_this", "tools[*].configs[*].id"],
+        json!({"tools": [
+            {"name": "t0", "configs": [{"remove_me": 1, "keep": 1}, {"keep": 2}], "metadata": {"drop_this": 1, "preserve": 1}},
+            {"name": "t1", "configs": [{"remove_me": 3, "keep": 3}, {"keep": 4}], "metadata": {"preserve": 2}},
+            {"name": "t2", "configs": [{"remove_me": 5, "keep": 5}], "metadata": {"drop_this": 3, "preserve": 3}}
+        ]}),
+    )]
+    #[case::simple_and_wildcard_nesting(
+        json!({
+            "tools": [{"name": "t1", "simple_nested": {"remove": 1, "keep": 2}, "complex": [{"nested": {"remove": 3, "keep": 4}}]}],
+            "top_level_remove": "should_go",
+            "top_level_keep": "should_stay"
+        }),
+        &["tools[*].simple_nested.remove", "tools[*].complex[*].nested.remove"],
+        json!({
+            "tools": [{"name": "t1", "simple_nested": {"keep": 2}, "complex": [{"nested": {"keep": 4}}]}],
+            "top_level_remove": "should_go",
+            "top_level_keep": "should_stay"
+        }),
+    )]
+    #[case::triple_nested_wildcards(
+        json!({"tools": [{"name": "t1", "arr1": [
+            {"arr2": [{"field": 1, "keep": 1}, {"field": 2, "keep": 2}]},
+            {"arr2": [{"field": 3, "keep": 3}]}
+        ]}]}),
+        &["tools[*].arr1[*].arr2[*].field"],
+        json!({"tools": [{"name": "t1", "arr1": [
+            {"arr2": [{"keep": 1}, {"keep": 2}]},
+            {"arr2": [{"keep": 3}]}
+        ]}]}),
+    )]
+    fn applies_paths_in_sequence(
+        #[case] value: Value,
+        #[case] paths: &[&str],
+        #[case] expected: Value,
+    ) {
+        let deleted = paths
+            .iter()
+            .fold(value, |value, path| delete_nested_value(value, path));
+        assert_eq!(deleted, expected);
+    }
+}

litellm-rust/crates/core-utils/src/get_provider_specific_headers.rs

@@ -0,0 +1,93 @@
+use litellm_types::utils::{ProviderSpecificHeader, ProviderSpecificHeaders};
+use serde_json::{Map, Value};
+
+pub fn get_provider_specific_headers(
+    provider_specific_header: Option<&ProviderSpecificHeaders>,
+    custom_llm_provider: &str,
+) -> Map<String, Value> {
+    let entries: &[ProviderSpecificHeader] = match provider_specific_header {
+        None => &[],
+        Some(ProviderSpecificHeaders::One(entry)) => std::slice::from_ref(entry),
+        Some(ProviderSpecificHeaders::Many(entries)) => entries,
+    };
+    entries
+        .iter()
+        .filter(|entry| {
+            entry
+                .custom_llm_provider
+                .split(',')
+                .any(|scoped| scoped.trim() == custom_llm_provider)
+        })
+        .flat_map(|entry| entry.extra_headers.clone())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::single_entry_for_the_provider(
+        json!({"custom_llm_provider": "anthropic", "extra_headers": {"Authorization": "Bearer t", "Custom-Header": "v"}}),
+        json!({"Authorization": "Bearer t", "Custom-Header": "v"}),
+    )]
+    #[case::single_entry_for_another_provider(
+        json!({"custom_llm_provider": "openai", "extra_headers": {"Authorization": "Bearer t"}}),
+        json!({}),
+    )]
+    #[case::provider_in_a_comma_separated_scope(
+        json!({"custom_llm_provider": "bedrock,anthropic,vertex_ai", "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}}),
+        json!({"anthropic-beta": "context-1m-2025-08-07"}),
+    )]
+    #[case::provider_missing_from_a_comma_separated_scope(
+        json!({"custom_llm_provider": "bedrock,vertex_ai", "extra_headers": {"anthropic-beta": "test"}}),
+        json!({}),
+    )]
+    #[case::scope_with_spaces(
+        json!({"custom_llm_provider": "bedrock, anthropic , vertex_ai", "extra_headers": {"anthropic-beta": "test"}}),
+        json!({"anthropic-beta": "test"}),
+    )]
+    #[case::scope_names_must_match_exactly(
+        json!({"custom_llm_provider": "anthropic_text", "extra_headers": {"anthropic-beta": "test"}}),
+        json!({}),
+    )]
+    #[case::entries_scope_independently(
+        json!([
+            {"custom_llm_provider": "anthropic,bedrock,vertex_ai", "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}},
+            {"custom_llm_provider": "bedrock", "extra_headers": {"x-bedrock-only": "no"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"authorization": "Bearer sk-ant-oat01-fake-token"}}
+        ]),
+        json!({"anthropic-beta": "context-1m-2025-08-07", "authorization": "Bearer sk-ant-oat01-fake-token"}),
+    )]
+    #[case::later_entries_win(
+        json!([
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "first"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "second"}}
+        ]),
+        json!({"x-scoped": "second"}),
+    )]
+    #[case::empty_list(json!([]), json!({}))]
+    #[case::entry_without_scope(json!({"extra_headers": {"x-scoped": "yes"}}), json!({}))]
+    #[case::entry_without_headers(json!({"custom_llm_provider": "anthropic"}), json!({}))]
+    fn provider_specific_headers_match_the_scoped_provider(
+        #[case] configured: Value,
+        #[case] expected: Value,
+    ) {
+        let configured: ProviderSpecificHeaders = serde_json::from_value(configured).unwrap();
+        assert_eq!(
+            Value::Object(get_provider_specific_headers(
+                Some(&configured),
+                "anthropic"
+            )),
+            expected
+        );
+    }
+
+    #[test]
+    fn no_configured_headers_match_nothing() {
+        assert_eq!(get_provider_specific_headers(None, "anthropic"), Map::new());
+    }
+}

litellm-rust/crates/core-utils/src/lib.rs

@@ -1,7 +1,9 @@
 pub mod call_arguments;
 pub mod core_helpers;
+pub mod dot_notation_indexing;
 pub mod exception_mapping_utils;
 pub mod get_llm_provider_logic;
+pub mod get_provider_specific_headers;
 pub mod params;
 pub mod prompt_templates;
 pub mod secret_redaction;

litellm-rust/crates/core-utils/src/prompt_templates/factory.rs

@@ -11,11 +11,13 @@
 //! accepts; anything richer is declined upstream by the capability gate.
 
 use litellm_types::llms::openai::{ChatMessage, ChatMessageContent};
+use strum::IntoStaticStr;
 
 pub const EMPTY_TEXT_PLACEHOLDER: &str =
     "[System: Empty message content sanitised to satisfy protocol]";
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq)]
+#[strum(serialize_all = "snake_case")]
 pub enum TurnRole {
     User,
     Assistant,
@@ -23,10 +25,7 @@ pub enum TurnRole {
 
 impl TurnRole {
     pub fn as_str(self) -> &'static str {
-        match self {
-            Self::User => "user",
-            Self::Assistant => "assistant",
-        }
+        self.into()
     }
 }
 

litellm-rust/crates/core/AGENTS.md

@@ -1,4 +1,6 @@
-litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-level call is a module under `src/<route>/` exposing a public entrypoint named after the route (`messages::messages()`, the Rust equivalent of `litellm.messages()`): you call it and get a typed non-streaming response back.
+litellm-core is the LiteLLM SDK in Rust. Each top-level call is a module under `src/<route>/` exposing a public entrypoint named after the route. `messages::messages()` returns `MessagesResponse::Message` for a completed response or `MessagesResponse::Stream { headers, chunks }` when the request sets `stream: true`. The chunks are Anthropic SSE bytes in a `Stream<Item = Result<Bytes, Error>>`. Dropping the stream cancels the call. The Python bridge drives `messages::route::messages_machine()` instead, because Python has to answer the call's operations on its own thread; the gateway and the Rust SDK call the plain entrypoint
+
+A route module has the same five pieces, in the order Python runs them. `types.rs` holds the call, the provider request, and the response. `prepare.rs` resolves the provider and credentials and shapes the request (Python's `validate_environment`, `get_complete_url`, `transform_request`). `handler.rs` resolves auth, offers the wire request to `litellm_host::hooks::RouteHooks::before_send`, sends it, reports the raw response through `emit`, and normalizes the response or stream (`pre_call`, `post`, `post_call`, `transform_response`). `mod.rs` exposes the entrypoint that runs prepare then handler with no hooks (`()`). `route.rs`, where a host needs it, wraps the same two calls in a `CallMachine` whose `HostChannel` is the hooks, and pumps a stream through `open` and `deliver`. A handler takes `&impl RouteHooks<Error>` and never a `HostChannel` directly, so it runs without a coroutine. Keep provider transport and transformation details out of the machine driver
 
 ## Crate layering
 
@@ -10,6 +12,16 @@ Each crate mirrors one top-level Python package, so a Rust path reads as its Pyt
 - `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `base_llm/ocr/handler.rs` (the OCR request handler)
 - `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks
 
-A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
+A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, the provider-level hooks OCR implements over its host until it folds into `litellm_host::hooks::RouteHooks`. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
+
+## Error placement
+
+The workspace `Error definitions` rules shape each crate's error; this section decides which crate and module a failure belongs to
+
+A failure is declared once, by the lowest crate that raises it. Every crate above nests that error unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`. `RouteError` collects route failures and never re-declares a variant a lower crate raises
+
+Scope follows the concept, not the first caller. An error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it. A failure two providers or two routes can hit, such as wire framing, stream event decoding, or a malformed provider response, belongs to the crate that owns the concept: `litellm-framing` for framing, `litellm_llms::Error` for the transformation layer
+
+`litellm_llms::Error` (`crates/llms/src/error.rs`) is the one transformation error for every provider and API. `base_llm/ocr/error.rs` is the recorded exception until OCR folds into it
 
 Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business.

litellm-rust/crates/core/Cargo.toml

@@ -4,7 +4,6 @@ version = "0.1.0"
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-autotests = false
 
 [dependencies]
 litellm-secrets.workspace = true
@@ -14,10 +13,11 @@ litellm-host.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 base64.workspace = true
-litellm-auth.workspace = true
+litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
 litellm-auth-aws.workspace = true
 litellm-http.workspace = true
 litellm-llms.workspace = true
+litellm-tracing.workspace = true
 moka.workspace = true
 mime_guess = "2.0.5"
 rand.workspace = true
@@ -37,7 +37,9 @@ url.workspace = true
 veil.workspace = true
 
 [dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
 litellm-auth-gcp.workspace = true
 litellm-llms = { workspace = true, features = ["test-support"] }
 rstest.workspace = true
 rstest_reuse.workspace = true
+wiremock = "0.6.5"

litellm-rust/crates/core/src/audio_transcription/client.rs

@@ -1,13 +0,0 @@
-use std::{sync::OnceLock, time::Duration};
-
-use crate::constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS;
-
-pub(super) fn http_client() -> &'static reqwest::Client {
-    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
-    CLIENT.get_or_init(|| {
-        reqwest::Client::builder()
-            .timeout(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS))
-            .build()
-            .unwrap_or_else(|_| reqwest::Client::new())
-    })
-}

litellm-rust/crates/core/src/audio_transcription/error.rs

@@ -1,43 +0,0 @@
-use litellm_llms::base_llm::chat::transformation::Error as LlmError;
-
-#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
-pub enum Error {
-    #[error("expected {expected}, got {actual}")]
-    InvalidType {
-        expected: &'static str,
-        actual: &'static str,
-    },
-    #[error("missing required field: {0}")]
-    MissingField(&'static str),
-    #[error("invalid provider: {0}")]
-    InvalidProvider(String),
-    #[error("invalid request: {0}")]
-    InvalidRequest(String),
-    #[error("invalid response: {0}")]
-    InvalidResponse(String),
-    #[error("unsupported by the rust path: {0}")]
-    Unsupported(&'static str),
-    #[error(transparent)]
-    Auth(#[from] litellm_auth::Error),
-    #[error(transparent)]
-    Transport(#[from] litellm_http::transport::Error),
-    #[error(transparent)]
-    Headers(#[from] litellm_http::request::HeaderError),
-    #[error(transparent)]
-    Http(#[from] litellm_http::Error),
-    #[error(transparent)]
-    Aws(#[from] litellm_auth_aws::Error),
-}
-
-impl From<LlmError> for Error {
-    fn from(error: LlmError) -> Self {
-        match error {
-            LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
-            LlmError::MissingField(field) => Self::MissingField(field),
-            LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
-            LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
-            LlmError::Unsupported(reason) => Self::Unsupported(reason),
-            LlmError::Auth(error) => Self::Auth(error),
-        }
-    }
-}

litellm-rust/crates/core/src/audio_transcription/handler.rs

@@ -1,22 +1,33 @@
-use litellm_http::request::truncate_error_body;
+use std::time::Duration;
+
+use litellm_http::{Client, request::truncate_error_body};
+use litellm_llms::base_llm::auth::resolve_auth;
 use serde_json::Value;
 
-use super::{Error, client::http_client};
-use crate::audio_transcription::types::ProviderAudioTranscriptionRequest;
+use super::Error;
+use crate::{
+    audio_transcription::types::ProviderAudioTranscriptionRequest,
+    constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS,
+};
 
 pub async fn execute_audio_transcription_provider_call(
+    http: &Client,
+    auth: &litellm_auth::AuthServices,
     request: ProviderAudioTranscriptionRequest,
 ) -> Result<Value, Error> {
-    let response = crate::outbound::outbound_request::<Error>(
-        &request.auth,
+    let env_lookup = |key: &str| std::env::var(key).ok();
+    let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
+    let response = crate::outbound::outbound_request(
+        authenticated,
         request.url.clone(),
-        request.upstream_headers.clone(),
         &request.body,
-        request.timeout,
-        &request.optional_params,
-    )
-    .await?
-    .send(http_client())
+        Some(
+            request
+                .timeout
+                .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
+        ),
+    )?
+    .send(http)
     .await
     .map_err(|error| {
         Error::Transport(litellm_http::transport::Error::Network(error.to_string()))

litellm-rust/crates/core/src/audio_transcription/mod.rs

@@ -1,19 +1,20 @@
-mod error;
 pub mod types;
-pub use error::Error;
-mod client;
+pub use crate::error::RouteError as Error;
 mod handler;
 mod prepare;
 pub use handler::execute_audio_transcription_provider_call;
+use litellm_http::{ClientVariant, HttpClientConfig};
 pub use prepare::prepare_audio_transcription_provider_call;
 use serde_json::Value;
 
 use crate::audio_transcription::types::AudioTranscriptionRequest;
 
-pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
-    execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?)
-        .await
+pub async fn audio_transcription(
+    resources: &crate::resources::CoreResources,
+    config: &HttpClientConfig,
+    request: AudioTranscriptionRequest<'_>,
+) -> Result<Value, Error> {
+    let request = prepare_audio_transcription_provider_call(request)?;
+    let http = resources.pool.client(config, ClientVariant::Provider)?;
+    execute_audio_transcription_provider_call(&http, &resources.auth, request).await
 }
-
-#[cfg(test)]
-mod tests;

litellm-rust/crates/core/src/audio_transcription/prepare.rs

@@ -1,7 +1,10 @@
 use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
-use litellm_http::request::{has_header, string_headers};
+use litellm_http::request::string_headers;
 use litellm_llms::{
-    base_llm::audio_transcription::transformation::{BaseAudioTranscriptionConfig, RequestAuth},
+    base_llm::{
+        audio_transcription::transformation::BaseAudioTranscriptionConfig,
+        auth::{ValidatedEnvironment, with_default_headers},
+    },
     bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
 };
 
@@ -39,20 +42,13 @@ pub fn prepare_audio_transcription_provider_call(
     let config = provider_config(provider_info.custom_llm_provider)
         .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?;
     let env_lookup = |key: &str| std::env::var(key).ok();
-    let mut headers = string_headers("audio transcription", request.extra_headers)?;
-    let auth = config.auth_strategy(&model, &request.optional_params, &env_lookup)?;
-    match &auth {
-        RequestAuth::Bearer { token } if !has_header(&headers, "authorization") => {
-            headers.push(("Authorization".to_string(), format!("Bearer {token}")));
-        }
-        RequestAuth::Header { name, value } if !has_header(&headers, name) => {
-            headers.push(((*name).to_string(), value.clone()));
-        }
-        RequestAuth::Bearer { .. } | RequestAuth::Header { .. } | RequestAuth::AwsSigV4 { .. } => {}
-    }
-    if !has_header(&headers, "content-type") {
-        headers.push(("Content-Type".to_string(), "application/json".to_string()));
-    }
+    let forwarded = string_headers("audio transcription", request.extra_headers)?;
+    let validated =
+        config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, &[("Content-Type", "application/json")]),
+        auth: validated.auth,
+    };
     let url = config.get_complete_url(
         request.api_base,
         &model,
@@ -68,9 +64,7 @@ pub fn prepare_audio_transcription_provider_call(
         config,
         url,
         body: transformed.body,
-        upstream_headers: headers,
-        auth,
-        optional_params: request.optional_params,
+        environment,
         timeout: request.timeout,
     })
 }

litellm-rust/crates/core/src/audio_transcription/tests.rs

@@ -1,52 +0,0 @@
-use std::{
-    io::{Read, Write},
-    net::TcpListener,
-    thread,
-};
-
-use serde_json::{Map, json};
-
-use super::audio_transcription;
-use crate::audio_transcription::types::AudioTranscriptionRequest;
-
-#[tokio::test]
-async fn bedrock_request_is_signed_and_contains_audio() {
-    let listener = TcpListener::bind("127.0.0.1:0").expect("listener");
-    let address = listener.local_addr().expect("address");
-    let server = thread::spawn(move || {
-        let (mut stream, _) = listener.accept().expect("connection");
-        let mut request = Vec::new();
-        let mut buffer = [0_u8; 16_384];
-        let count = stream.read(&mut buffer).expect("request");
-        request.extend_from_slice(&buffer[..count]);
-        let request = String::from_utf8_lossy(&request);
-        assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse"));
-        assert!(request.contains("authorization: AWS4-HMAC-SHA256"));
-        assert!(request.contains("x-amz-date:"));
-        assert!(request.contains("\"bytes\":\"AQI=\""));
-        assert!(request.contains("Transcribe the audio. Respond with only the transcript."));
-        let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}";
-        stream.write_all(response).expect("response");
-    });
-
-    let optional_params = Map::from_iter([
-        ("aws_access_key_id".to_string(), json!("access-key")),
-        ("aws_secret_access_key".to_string(), json!("secret-key")),
-        ("aws_region_name".to_string(), json!("us-east-1")),
-    ]);
-    let api_base = format!("http://{address}");
-    let response = audio_transcription(AudioTranscriptionRequest {
-        model: "mistral.voxtral-mini-3b-2507",
-        audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}),
-        api_key: None,
-        api_base: Some(&api_base),
-        custom_llm_provider: Some("bedrock"),
-        extra_headers: None,
-        optional_params,
-        timeout: None,
-    })
-    .await
-    .expect("transcription");
-    assert_eq!(response, json!({"text": "hello"}));
-    server.join().expect("server");
-}

litellm-rust/crates/core/src/audio_transcription/types.rs

@@ -1,7 +1,7 @@
 use std::time::Duration;
 
-use litellm_llms::base_llm::audio_transcription::transformation::{
-    BaseAudioTranscriptionConfig, RequestAuth,
+use litellm_llms::base_llm::{
+    audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
 };
 use serde_json::{Map, Value};
 
@@ -23,9 +23,7 @@ pub struct ProviderAudioTranscriptionRequest {
     pub config: &'static dyn BaseAudioTranscriptionConfig,
     pub url: String,
     pub body: Value,
-    pub upstream_headers: Vec<(String, String)>,
-    pub auth: RequestAuth,
-    pub optional_params: Map<String, Value>,
+    pub environment: ValidatedEnvironment,
     pub timeout: Option<Duration>,
 }
 

litellm-rust/crates/core/src/chat_completions/client.rs

@@ -1,14 +0,0 @@
-use std::{sync::OnceLock, time::Duration};
-
-use crate::constants::{CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS, CHAT_COMPLETIONS_TIMEOUT_SECS};
-
-pub(super) fn http_client() -> &'static reqwest::Client {
-    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
-    CLIENT.get_or_init(|| {
-        reqwest::Client::builder()
-            .timeout(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS))
-            .connect_timeout(Duration::from_secs(CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS))
-            .build()
-            .unwrap_or_else(|_| reqwest::Client::new())
-    })
-}

litellm-rust/crates/core/src/chat_completions/common_utils.rs

@@ -3,6 +3,7 @@ use litellm_llms::{
     anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
     base_llm::chat::transformation::BaseConfig,
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
+    openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
 use serde_json::{Map, Value};
 
@@ -14,6 +15,7 @@ pub(super) fn chat_completions_provider_config(provider: &str) -> Option<&'stati
     match provider {
         "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG),
         "bedrock" => Some(&BEDROCK_CHAT_COMPLETIONS_CONFIG),
+        "openai_like" => Some(&OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG),
         _ => None,
     }
 }

litellm-rust/crates/core/src/chat_completions/error.rs

@@ -1,43 +0,0 @@
-use litellm_llms::base_llm::chat::transformation::Error as LlmError;
-
-#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
-pub enum Error {
-    #[error("expected {expected}, got {actual}")]
-    InvalidType {
-        expected: &'static str,
-        actual: &'static str,
-    },
-    #[error("missing required field: {0}")]
-    MissingField(&'static str),
-    #[error("invalid provider: {0}")]
-    InvalidProvider(String),
-    #[error("invalid request: {0}")]
-    InvalidRequest(String),
-    #[error("invalid response: {0}")]
-    InvalidResponse(String),
-    #[error("unsupported by the rust path: {0}")]
-    Unsupported(&'static str),
-    #[error(transparent)]
-    Auth(#[from] litellm_auth::Error),
-    #[error(transparent)]
-    Transport(#[from] litellm_http::transport::Error),
-    #[error(transparent)]
-    Headers(#[from] litellm_http::request::HeaderError),
-    #[error(transparent)]
-    Http(#[from] litellm_http::Error),
-    #[error(transparent)]
-    Aws(#[from] litellm_auth_aws::Error),
-}
-
-impl From<LlmError> for Error {
-    fn from(error: LlmError) -> Self {
-        match error {
-            LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
-            LlmError::MissingField(field) => Self::MissingField(field),
-            LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
-            LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
-            LlmError::Unsupported(reason) => Self::Unsupported(reason),
-            LlmError::Auth(error) => Self::Auth(error),
-        }
-    }
-}

litellm-rust/crates/core/src/chat_completions/handler.rs

@@ -1,20 +1,70 @@
-use litellm_http::{outbound::OutboundRequest, request::truncate_error_body};
-use litellm_llms::base_llm::chat::transformation::ProviderChatResponseData;
+use std::time::Duration;
+
+use litellm_auth::AuthServices;
+use litellm_host::{
+    event::{MachineEvent, RawResponse, RequestContext, WireRequest},
+    hooks::RouteHooks,
+};
+use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
+use litellm_llms::base_llm::{
+    auth::{Authenticated, resolve_auth},
+    chat::transformation::ProviderChatResponseData,
+};
 use litellm_types::utils::ChatCompletionsResponse;
 use serde_json::Value;
 
-use super::{Error, client::http_client, prepare::prepare_provider_request};
-use crate::chat_completions::types::{
-    ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
+use super::Error;
+use crate::{
+    chat_completions::types::ProviderChatCompletionsRequest,
+    constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
 };
 
-pub(super) async fn execute_chat_completions_provider_call(
-    request: ResolvedChatCompletionsRequest<'_>,
+pub(super) async fn execute(
+    http: &Client,
+    auth: &AuthServices,
+    request: ProviderChatCompletionsRequest,
+    hooks: &impl RouteHooks<Error>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    let request = prepare_provider_request(request)?;
-    let outbound = outbound_request(&request).await?;
+    let ProviderChatCompletionsRequest {
+        model,
+        custom_llm_provider,
+        config,
+        url,
+        body,
+        optional_params,
+        environment,
+        timeout,
+        api_key,
+    } = request;
+    let context = RequestContext {
+        model: model.clone(),
+        custom_llm_provider,
+        optional_params: Value::Object(optional_params),
+        secret_fields: Vec::new(),
+        api_key,
+    };
+    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
+    let wire = hooks
+        .before_send(
+            WireRequest {
+                url,
+                headers: authenticated.headers,
+                body,
+            },
+            context,
+        )
+        .await?;
+    let outbound = outbound_request(
+        Authenticated {
+            headers: wire.headers,
+            signer: authenticated.signer,
+        },
+        wire.url,
+        &wire.body,
+        timeout,
+    )?;
 
-    let response = outbound.send(http_client()).await.map_err(|err| {
+    let response = outbound.send(http).await.map_err(|err| {
         // Failing to establish the connection means the request never went out,
         // so the host can still serve it. Everything else here, a timeout
         // above all, may have reached the provider and been answered.
@@ -36,13 +86,17 @@ pub(super) async fn execute_chat_completions_provider_call(
             body: truncate_error_body(&text),
         }));
     }
+    hooks
+        .emit(MachineEvent::ResponseReceived {
+            raw: RawResponse { body: text.clone() },
+        })
+        .await?;
 
     let body: Value = serde_json::from_str(&text).map_err(|err| {
         Error::InvalidResponse(format!("invalid chat completions response JSON: {err}"))
     })?;
-    request
-        .config
-        .transform_response(&request.model, ProviderChatResponseData { body })
+    config
+        .transform_response(&model, ProviderChatResponseData { body })
         .map_err(Error::from)
         .map_err(as_response_error)
 }
@@ -64,24 +118,176 @@ pub(super) fn as_response_error(err: Error) -> Error {
     }
 }
 
-pub(super) async fn outbound_request(
-    request: &ProviderChatCompletionsRequest,
+pub(super) fn outbound_request(
+    authenticated: Authenticated,
+    url: String,
+    body: &Value,
+    timeout: Option<Duration>,
 ) -> Result<OutboundRequest, Error> {
     crate::outbound::outbound_request(
-        &request.auth,
-        request.url.clone(),
-        request.upstream_headers.clone(),
-        &request.body,
-        request.timeout,
-        &request.optional_params,
+        authenticated,
+        url,
+        body,
+        Some(timeout.unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS))),
     )
-    .await
     .map_err(|error| match error {
         // Python drops the caller's copy and prefers a forwarded Authorization
         // over the signature, so leave the request to it.
-        Error::Http(litellm_http::Error::ComputedHeader(_)) => {
+        litellm_http::Error::ComputedHeader(_) => {
             Error::Unsupported("request forwards a header AWS SigV4 computes")
         }
-        other => other,
+        other => Error::Http(other),
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Mutex;
+
+    use rstest::rstest;
+    use serde_json::json;
+    use wiremock::{Mock, MockServer, Request, ResponseTemplate, matchers::any};
+
+    use super::*;
+    use crate::chat_completions::{
+        prepare::{prepare_provider_request, resolve_request},
+        types::ChatCompletionsRequest,
+    };
+
+    const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+    /// Rewrites the outgoing request and records what the call reports back.
+    #[derive(Default)]
+    struct RecordingHooks {
+        contexts: Mutex<Vec<RequestContext>>,
+        raw: Mutex<Vec<String>>,
+    }
+
+    impl RouteHooks<Error> for RecordingHooks {
+        async fn before_send(
+            &self,
+            wire: WireRequest,
+            context: RequestContext,
+        ) -> Result<WireRequest, Error> {
+            self.contexts.lock().unwrap().push(context);
+            let mut body = wire.body;
+            body["system"] = json!("added by the host");
+            Ok(WireRequest {
+                headers: wire
+                    .headers
+                    .into_iter()
+                    .chain([("x-host".to_string(), "seen".to_string())])
+                    .collect(),
+                body,
+                ..wire
+            })
+        }
+
+        async fn emit(&self, event: MachineEvent) -> Result<(), Error> {
+            let MachineEvent::ResponseReceived { raw } = event;
+            self.raw.lock().unwrap().push(raw.body);
+            Ok(())
+        }
+    }
+
+    fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
+        prepare_provider_request(
+            resolve_request(ChatCompletionsRequest {
+                model: "anthropic/claude-sonnet-4-5",
+                messages: json!([{"role": "user", "content": "hi"}]),
+                optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
+                api_key: Some("sk-test"),
+                api_base: Some(api_base),
+                custom_llm_provider: None,
+                extra_headers: None,
+                timeout: None,
+            })
+            .unwrap(),
+        )
+        .unwrap()
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn the_hooks_rewrite_the_wire_request_and_see_the_raw_response() {
+        let upstream = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(
+                ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json"),
+            )
+            .mount(&upstream)
+            .await;
+        let hooks = RecordingHooks::default();
+
+        execute(
+            &Client::plain_for_test(),
+            &AuthServices::default(),
+            prepared(&upstream.uri()),
+            &hooks,
+        )
+        .await
+        .expect("chat completions call succeeds");
+
+        let [request] = <[Request; 1]>::try_from(upstream.received_requests().await.unwrap())
+            .unwrap_or_else(|requests| panic!("one request, saw {}", requests.len()));
+        let sent: Value = serde_json::from_slice(&request.body).unwrap();
+        assert_eq!(sent["system"], "added by the host");
+        assert_eq!(request.headers["x-host"], "seen");
+        assert_eq!(request.headers["x-api-key"], "sk-test");
+        let [context] = <[RequestContext; 1]>::try_from(hooks.contexts.into_inner().unwrap())
+            .unwrap_or_else(|seen| panic!("before_send runs once, saw {}", seen.len()));
+        assert_eq!(
+            (context.model.as_str(), context.custom_llm_provider.as_str()),
+            ("claude-sonnet-4-5", "anthropic")
+        );
+        assert_eq!(context.optional_params, json!({"max_tokens": 16}));
+        assert_eq!(hooks.raw.into_inner().unwrap(), [ANTHROPIC_MESSAGE]);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn an_upstream_failure_is_not_reported_as_a_received_response() {
+        let upstream = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(ResponseTemplate::new(500).set_body_string("boom"))
+            .mount(&upstream)
+            .await;
+        let hooks = RecordingHooks::default();
+
+        let error = execute(
+            &Client::plain_for_test(),
+            &AuthServices::default(),
+            prepared(&upstream.uri()),
+            &hooks,
+        )
+        .await
+        .expect_err("the upstream failure fails the call");
+
+        assert!(matches!(
+            error,
+            Error::Transport(litellm_http::transport::Error::Http { status: 500, .. })
+        ));
+        assert!(hooks.raw.into_inner().unwrap().is_empty());
+    }
+
+    #[test]
+    fn response_errors_collapse_to_one_variant_that_can_only_mean_already_sent() {
+        for original in [
+            Error::MissingField("usage"),
+            Error::Unsupported("non-text response content block"),
+            Error::InvalidRequest("whatever".to_string()),
+            Error::Auth(litellm_auth::Error::InvalidHeader),
+        ] {
+            let label = format!("{original:?}");
+            assert!(
+                matches!(as_response_error(original), Error::InvalidResponse(_)),
+                "{label} must not stay retryable once the provider has answered"
+            );
+        }
+        let upstream = Error::Transport(litellm_http::transport::Error::Http {
+            status: 500,
+            body: "boom".to_string(),
+        });
+        assert_eq!(as_response_error(upstream.clone()), upstream);
+    }
+}

litellm-rust/crates/core/src/chat_completions/mod.rs

@@ -6,24 +6,26 @@
 //! credentials, and it resolves the provider, translates the conversation,
 //! calls the provider, and returns a typed OpenAI-shaped response.
 
-mod error;
 pub mod types;
-pub use error::Error;
-mod client;
+pub use crate::error::RouteError as Error;
 mod common_utils;
 pub(crate) mod handler;
 mod prepare;
-use handler::execute_chat_completions_provider_call;
+use litellm_http::{ClientVariant, HttpClientConfig};
 use litellm_types::utils::ChatCompletionsResponse;
-use prepare::{parse_messages, resolve_provider_config, resolve_request};
+use prepare::{parse_messages, prepare_provider_request, resolve_provider_config, resolve_request};
 use serde_json::{Map, Value};
 
 use crate::chat_completions::types::ChatCompletionsRequest;
 
 pub async fn chat_completions(
+    resources: &crate::resources::CoreResources,
+    config: &HttpClientConfig,
     request: ChatCompletionsRequest<'_>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    execute_chat_completions_provider_call(resolve_request(request)?).await
+    let http = resources.pool.client(config, ClientVariant::Provider)?;
+    let request = prepare_provider_request(resolve_request(request)?)?;
+    handler::execute(&http, &resources.auth, request, &()).await
 }
 
 /// Whether the core would accept this request, without resolving credentials or
@@ -38,9 +40,10 @@ pub fn chat_completions_decline_reason(
     messages: Value,
     optional_params: &Map<String, Value>,
 ) -> Option<&'static str> {
-    let Ok((_, config)) = resolve_provider_config(model, custom_llm_provider) else {
+    let Ok(resolved) = resolve_provider_config(model, custom_llm_provider) else {
         return Some("provider is not on the rust chat completions path");
     };
+    let config = resolved.config;
     let Ok(messages) = parse_messages(messages) else {
         return Some("unreadable message list");
     };
@@ -51,6 +54,3 @@ pub fn chat_completions_decline_reason(
         .unsupported_reason(&messages, optional_params)
         .map(|reason| reason.0)
 }
-
-#[cfg(test)]
-mod tests;