Merge remote-tracking branch 'origin/main' into litellm_techdebt_20261010

.circleci/config.yml

@@ -38,7 +38,7 @@ commands:
     parameters:
       category:
         type: enum
-        enum: ["backend", "client", "provider-harness", "redis-compat"]
+        enum: ["backend", "client", "redis-compat"]
         default: "backend"
     steps:
       - run:
@@ -160,7 +160,7 @@ commands:
                 exit 1
                 ;;
             esac
-            curl -sSLf -o /tmp/rustup-init \
+            curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \
               "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
             echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
             chmod +x /tmp/rustup-init
@@ -453,105 +453,6 @@ jobs:
           command: |
             python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
 
-  llm_translation_testing:
-    docker:
-      - &python312_image
-        image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-    working_directory: ~/project
-    resource_class: xlarge
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v1-uv-cache-{{ checksum "uv.lock" }}
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v1-uv-cache-{{ checksum "uv.lock" }}
-      # Run pytest and generate JUnit XML report
-      - run:
-          name: Run tests
-          command: |
-            # Add --timeout to kill hanging tests after 120s (2 min)
-            # Add --durations=20 to show 20 slowest tests for debugging
-            # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
-            mkdir -p test-results
-            # Glob excludes the realtime/ subdirectory since it has its own job
-            TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-                -v \
-                --junitxml=test-results/junit.xml \
-                --durations=20 \
-                -n 4 \
-                --timeout=120 --timeout_method=thread \
-                --retries 2 --retry-delay 5 \
-                --max-worker-restart=5"
-          no_output_timeout: 15m
-
-      # Store test results
-      - store_test_results:
-          path: test-results
-  realtime_translation_testing:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      # Run pytest and generate JUnit XML report
-      - run:
-          name: Run realtime tests
-          command: |
-            # Add --timeout to kill hanging tests after 120s (2 min)
-            # Add --durations=20 to show 20 slowest tests for debugging
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-                -vv \
-                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-                --junitxml=test-results/junit.xml \
-                --durations=20 \
-                -n 4 \
-                --timeout=120 --timeout_method=thread"
-          no_output_timeout: 15m
-      - run:
-          name: Rename the coverage files
-          command: |
-            mv coverage.xml realtime_translation_coverage.xml
-            mv .coverage realtime_translation_coverage
-
-      # Store test results
-      - store_test_results:
-          path: test-results
-      - persist_to_workspace:
-          root: .
-          paths:
-            - realtime_translation_coverage.xml
-            - realtime_translation_coverage
   helm_chart_testing:
     machine:
       image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
@@ -718,7 +619,7 @@ jobs:
           name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
           command: |
             mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
@@ -795,7 +696,7 @@ jobs:
           name: Run second round of tests
           command: |
             mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
@@ -807,110 +708,46 @@ jobs:
       # Store test results
       - store_test_results:
           path: test-results
-  proxy_pass_through_endpoint_tests:
-    machine:
-      image: ubuntu-2204:2024.04.1
+  e2e_ui_build:
+    docker:
+      - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
     resource_class: large
     working_directory: ~/project
     steps:
       - checkout
-      - run:
-          name: Generate LiteLLM master key
-          command: |
-            key="$(openssl rand -hex 16)"
-            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
-      - skip_if_unrelated_changes
+      - skip_if_unrelated_changes:
+          category: client
       - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - start_postgres
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Load Docker Database Image
-          command: |
-            zstd -d litellm-docker-database.tar.zst --stdout | docker load
-            docker images | grep litellm-docker-database
-      - run:
-          name: Run Docker container
-          command: |
-            docker run -d \
-              -p 4000:4000 \
-              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-              -e OPENAI_API_KEY=$OPENAI_API_KEY \
-              -e GEMINI_API_KEY=$GEMINI_API_KEY \
-              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-              -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
-              -e AZURE_API_KEY=$AZURE_API_KEY \
-              -e AZURE_API_BASE=$AZURE_API_BASE \
-              -e USE_DDTRACE=True \
-              -e DD_API_KEY=$DD_API_KEY \
-              -e DD_SITE=$DD_SITE \
-              -e LITELLM_LICENSE=$LITELLM_LICENSE \
-              -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
-              -e LITELLM_LOG=ERROR \
-              --add-host host.docker.internal:host-gateway \
-              --name my-app \
-              -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-              -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
-              litellm-docker-database:ci \
-              --config /app/config.yaml \
-              --port 4000
-      - run:
-          name: Start outputting logs
-          command: docker logs -f my-app
-          background: true
-      - wait_for_service:
-          url: http://localhost:4000
-          timeout: "300"
-      # Install Node.js directly from nodejs.org with SHA256 verification,
-      # instead of piping NodeSource's setup_24.x apt-repo installer into
-      # sudo bash (which runs a mutable upstream script unattended).
       - install_node
-
+      - restore_cache:
+          keys:
+            - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
       - run:
-          name: Install Node.js test dependencies
+          name: Install dashboard dependencies
           command: |
-            cd tests/pass_through_tests
+            cd ui/litellm-dashboard
             npm ci
-
+      - save_cache:
+          key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          paths:
+            - ui/litellm-dashboard/node_modules
       - run:
-          name: Run Vertex AI, Google AI Studio Node.js tests
+          name: Build UI bundle
           command: |
-            cd tests/pass_through_tests
-            NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
-          no_output_timeout: 30m
-
-  upload-coverage:
-    docker:
-      - *python312_image
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - attach_workspace:
-          at: .
-      # Check file locations
-      - run:
-          name: Check coverage file location
-          command: |
-            echo "Current directory:"
-            ls -la
-            echo "\nContents of tests/llm_translation:"
-            ls -la tests/llm_translation
-      - install_uv
-      - run:
-          name: Combine Coverage
-          command: |
-            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage
-            uv tool run --from 'coverage[toml]==7.10.6' coverage xml
-      - codecov/upload:
-          file: ./coverage.xml
-          flags: circleci
+            cd ui/litellm-dashboard
+            npm run build
+            mkdir -p ../../ui-bundle
+            mv out ../../ui-bundle/out
+            find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do
+              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
+            done
+      - persist_to_workspace:
+          root: ~/project
+          paths:
+            - ui-bundle/out
 
   e2e_ui_testing:
     docker:
@@ -923,7 +760,7 @@ jobs:
           POSTGRES_USER: e2euser
           POSTGRES_PASSWORD: e2epassword
           POSTGRES_DB: litellm_e2e
-    resource_class: large
+    resource_class: xlarge
     working_directory: ~/project
     environment:
       DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
@@ -942,6 +779,8 @@ jobs:
             printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: client
+      - attach_workspace:
+          at: ~/project
       - setup_google_dns
       - install_node
       - install_uv
@@ -960,7 +799,7 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+            - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           # The cimg/python:3.12-browsers image already ships the Chromium system
@@ -969,36 +808,19 @@ jobs:
           # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
           # so we install just the browser binary.
           command: |
-            cd ui/litellm-dashboard
-            npm ci
-            cd ../../tests/e2e/ui
+            cd tests/e2e/ui
             npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+          key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
-            - ui/litellm-dashboard/node_modules
             - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
-          name: Build UI from source
-          # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
-          # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the
-          # source directory as a child of the destination" when the destination
-          # already exists — silently creating `_experimental/out/out/` instead of
-          # replacing the served bundle. The proxy continued serving whatever was
-          # checked into `_experimental/out/*`, so this job was effectively testing
-          # the pre-build bundle on every run. Replace-and-move guarantees the
-          # freshly built bundle is what the proxy actually serves.
+          name: Use built UI bundle
           command: |
-            cd ui/litellm-dashboard
-            npm run build
-            rm -rf ../../litellm/proxy/_experimental/out
-            mv out ../../litellm/proxy/_experimental/out
-            # Restructure HTML so extensionless routes work (login.html -> login/index.html)
-            find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
-              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
-            done
+            rm -rf litellm/proxy/_experimental/out
+            cp -r ui-bundle/out litellm/proxy/_experimental/out
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "30"
@@ -1065,7 +887,7 @@ jobs:
           command: |
             cd tests/e2e/ui
             LITELLM_LICENSE="$LITELLM_LICENSE" \
-              npx playwright test --config playwright.config.ts
+              npx playwright test --config playwright.config.ts --workers=3 --retries=0
           no_output_timeout: 10m
       - store_artifacts:
           path: tests/e2e/ui/test-results
@@ -1103,6 +925,8 @@ jobs:
             printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: client
+      - attach_workspace:
+          at: ~/project
       - setup_google_dns
       - install_node
       - install_uv
@@ -1121,31 +945,23 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+            - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           command: |
-            cd ui/litellm-dashboard
-            npm ci
-            cd ../../tests/e2e/ui
+            cd tests/e2e/ui
             npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+          key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
-            - ui/litellm-dashboard/node_modules
             - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
-          name: Build UI from source
+          name: Use built UI bundle
           command: |
-            cd ui/litellm-dashboard
-            npm run build
-            rm -rf ../../litellm/proxy/_experimental/out
-            mv out ../../litellm/proxy/_experimental/out
-            find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
-              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
-            done
+            rm -rf litellm/proxy/_experimental/out
+            cp -r ui-bundle/out litellm/proxy/_experimental/out
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "30"
@@ -1396,41 +1212,6 @@ jobs:
             ! docker exec my-app true 2>/dev/null || fail "container is still running"
             echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
 
-  provider_replay_harness:
-    docker:
-      - *python312_image
-      - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
-    working_directory: ~/project
-    resource_class: medium
-    environment:
-      E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0
-      E2E_PROVIDER_CACHE: "0"
-      E2E_FIXTURE_MODE: live
-    steps:
-      - checkout
-      - run:
-          name: Generate LiteLLM master key
-          command: |
-            key="$(openssl rand -hex 16)"
-            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
-      - skip_if_unrelated_changes:
-          category: provider-harness
-      - setup_litellm_test_deps
-      - wait_for_service:
-          url: tcp://localhost:6379
-      - run:
-          name: Test provider capture and replay harness
-          command: |
-            mkdir -p test-results/provider-replay-harness
-            uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
-              --junitxml=test-results/provider-replay-harness/junit.xml \
-              tests/e2e_harness/test_provider_edge.py tests/e2e_harness/test_fixture_bundle.py \
-              tests/e2e_harness/test_fixture_canonical.py tests/e2e_harness/test_fixture_mode.py \
-              tests/code_coverage_tests/test_provider_replay_harness.py \
-              tests/code_coverage_tests/test_provider_cache.py
-      - store_test_results:
-          path: test-results/provider-replay-harness
-
   integration_contracts:
     parameters:
       suite:
@@ -1479,6 +1260,16 @@ jobs:
             - run:
                 name: Build the candidate dashboard
                 command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
+      - when:
+          condition:
+            equal: [mcp, << parameters.suite >>]
+          steps:
+            - run:
+                name: Install MCP SDK1 peer
+                command: |
+                  uv venv --python 3.12 .venv-mcp-peer
+                  uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+                  echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
           server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
@@ -1727,7 +1518,7 @@ jobs:
             mkdir -p test-results
             env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
               timeout --signal=TERM 20m uv run --no-sync pytest \
-              tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
+              tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
               tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
               --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
               --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
@@ -1803,9 +1594,14 @@ jobs:
           path: test-results
 
 workflows:
-  migration_startup:
-    when: << pipeline.parameters.run_migration_tests >>
-    jobs: &migration_jobs
+  migration_startup_scheduled:
+    triggers:
+      - schedule:
+          cron: "17 0,6,12,18 * * *"
+          filters:
+            branches:
+              only: main
+    jobs:
       - build_docker_database_image:
           migration_qualification: true
       - migration_startup_tests:
@@ -1830,33 +1626,11 @@ workflows:
           suite: shaped
           baseline: true
           requires: [build_docker_database_image]
-  migration_startup_scheduled:
-    triggers:
-      - schedule:
-          cron: "17 0,6,12,18 * * *"
-          filters:
-            branches:
-              only: main
-    jobs: *migration_jobs
-  routing_parity:
-    when:
-      not:
-        equal: ["", << pipeline.parameters.routing_parity_base >>]
-    jobs:
-      - routing_parity:
-          name: routing-parity-<< matrix.suite >>
-          matrix:
-            parameters:
-              suite: [management, accounting, database, providers, extensions, cost, mcp]
   integration:
-    unless:
-      or:
-        - << pipeline.parameters.run_migration_tests >>
-        - not:
-            equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
       - integration_contracts:
           name: integration-<< matrix.suite >>
+          filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == ""
           matrix:
             parameters:
               suite: [sdk, cost, browser]
@@ -1866,111 +1640,118 @@ workflows:
           suite: security
           parallelism: 1
           resource_class: xlarge
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-management
           suite: management
           parallelism: 4
+          resource_class: xlarge
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-management-coverage
           suite: management
           shards: 4
           requires: [integration-management]
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-accounting
           suite: accounting
           parallelism: 4
+          resource_class: xlarge
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-accounting-coverage
           suite: accounting
           shards: 4
           requires: [integration-accounting]
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-providers
           suite: providers
-          parallelism: 8
+          parallelism: 4
+          resource_class: xlarge
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-providers-coverage
           suite: providers
-          shards: 8
+          shards: 4
           requires:
             - integration-providers
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-database
           suite: database
           parallelism: 2
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-database-coverage
           suite: database
           shards: 2
           requires: [integration-database]
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-mcp
           suite: mcp
           resource_class: xlarge
           parallelism: 2
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-mcp-coverage
           suite: mcp
           shards: 2
           requires:
             - integration-mcp
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-extensions
           suite: extensions
           parallelism: 8
+          resource_class: xlarge
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-extensions-coverage
           suite: extensions
           shards: 8
           requires:
             - integration-extensions
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-management-replica
           suite: management
           mode: replica
           parallelism: 4
+          resource_class: xlarge
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-management-replica-coverage
           suite: management
           mode: replica
           shards: 4
           requires: [integration-management-replica]
+          filters: *integration_default_run
       - integration_contracts:
           name: integration-database-replica
           suite: database
           mode: replica
           parallelism: 2
+          filters: *integration_default_run
       - integration_shard_coverage:
           name: integration-database-replica-coverage
           suite: database
           mode: replica
           shards: 2
           requires: [integration-database-replica]
-      - postgres_suite:
-          name: proxy-behavior
-          test_path: tests/proxy_behavior
-          seed: true
-          coverage_flag: lens-postgres
-          timeout_minutes: 25
-      - postgres_suite:
-          name: proxy-security
-          test_path: tests/proxy_security_tests
-          seed: true
-          timeout_minutes: 15
-      - postgres_suite:
-          name: schema-migration
-          test_path: tests/proxy_migration_tests
-          seed: false
-          timeout_minutes: 20
-          workers: 2
+          filters: *integration_default_run
       - postgres_suite:
           name: roi-database
           test_path: tests/integration/database/test_roi_observed.py
           seed: false
           coverage_flag: roi-postgres
           timeout_minutes: 10
+          filters: *integration_default_run
       - mcp_integration:
           name: mcp-integration
+          filters: *integration_default_run
       - redis_compat:
           name: redis-compat-<< matrix.redis_py >>
           matrix:
@@ -1980,36 +1761,71 @@ workflows:
                 - "6.4.0"
                 - "7.4.1"
                 - "8.0.1"
-      - windows_release_wheel
-      - base_sdk_install
-  build_and_test:
-    unless:
-      or:
-        - << pipeline.parameters.run_migration_tests >>
-        - not:
-            equal: ["", << pipeline.parameters.routing_parity_base >>]
-    jobs:
-      - provider_replay_harness
-      - build_docker_database_image
-      - e2e_ui_testing
-      - e2e_ui_testing_server_root_path
+          filters: *integration_default_run
+      - windows_release_wheel:
+          filters: *integration_default_run
+      - base_sdk_install:
+          filters: *integration_default_run
+      - build_docker_database_image:
+          filters: *integration_default_run
       - proxy_logging_guardrails_model_info_tests:
           requires:
             - build_docker_database_image
-      - proxy_pass_through_endpoint_tests:
-          requires:
-            - build_docker_database_image
-      - llm_translation_testing
-      - realtime_translation_testing
-      - upload-coverage:
-          requires:
-            - realtime_translation_testing
+          filters: *integration_default_run
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-      - helm_chart_testing:
-          requires:
-            - build_docker_database_image
+          filters: *integration_default_run
       - test_bad_database_url:
           requires:
             - build_docker_database_image
+          filters: *integration_default_run
+      - helm_chart_testing:
+          requires:
+            - build_docker_database_image
+          filters: *integration_default_run
+      - e2e_ui_build:
+          filters: *integration_default_run
+      - e2e_ui_testing:
+          requires: [e2e_ui_build]
+          filters: *integration_default_run
+      - e2e_ui_testing_server_root_path:
+          requires: [e2e_ui_build]
+          filters: *integration_default_run
+      - build_docker_database_image:
+          name: build_docker_database_image-migration
+          migration_qualification: true
+          filters: pipeline.parameters.run_migration_tests
+      - migration_startup_tests:
+          name: migration-startup
+          suite: startup
+          requires: [build_docker_database_image-migration]
+          filters: pipeline.parameters.run_migration_tests
+      - migration_startup_tests:
+          name: migration-recovery
+          suite: recovery
+          requires: [build_docker_database_image-migration]
+          filters: pipeline.parameters.run_migration_tests
+      - migration_startup_tests:
+          name: migration-legacy-and-pooling
+          suite: legacy
+          requires: [build_docker_database_image-migration]
+          filters: pipeline.parameters.run_migration_tests
+      - migration_startup_tests:
+          name: migration-upgrade
+          suite: upgrade
+          baseline: true
+          requires: [build_docker_database_image-migration]
+          filters: pipeline.parameters.run_migration_tests
+      - migration_startup_tests:
+          name: migration-upgrade-shaped
+          suite: shaped
+          baseline: true
+          requires: [build_docker_database_image-migration]
+          filters: pipeline.parameters.run_migration_tests
+      - routing_parity:
+          name: routing-parity-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [management, accounting, database, providers, extensions, cost, mcp]
+          filters: pipeline.parameters.routing_parity_base != ""

.circleci/scripts/classify_changes.sh

@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
 
 has_client=false
 has_backend=false
 has_ci=false
-has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
 has_windows_release=false
@@ -23,11 +22,6 @@ while IFS= read -r file || [ -n "$file" ]; do
     pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
       has_mcp_dependencies=true ;;
   esac
-  case "$file" in
-    tests/e2e/*/*.py | tests/e2e_harness/*/*.py) : ;;
-    tests/e2e/*.py | tests/e2e_harness/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
-      has_provider_harness=true ;;
-  esac
   case "$file" in
     litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
       has_windows_release=true ;;
@@ -41,7 +35,7 @@ while IFS= read -r file || [ -n "$file" ]; do
   case "$file" in
     model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
       has_cost_map=true ;;
-    tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;;
+    tests/unit/proxy/*) : ;;
     *) outside_cost_map_set=true ;;
   esac
 done
@@ -53,9 +47,6 @@ case "$category" in
   cost-map-only)
     { [ "$has_cost_map" = true ] && [ "$outside_cost_map_set" = false ]; } && echo run || echo skip
     ;;
-  provider-harness)
-    [ "$has_provider_harness" = true ] && echo run || echo skip
-    ;;
   windows-release)
     [ "$has_windows_release" = true ] && echo run || echo skip
     ;;

.circleci/scripts/path_filter.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: path_filter.sh <backend|client|provider-harness|redis-compat>}"
+category="${1:?usage: path_filter.sh <backend|client|redis-compat>}"
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 run_full() {

.circleci/scripts/run_integration.sh

@@ -128,9 +128,20 @@ fi
 sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
 awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
 
-setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
-  .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
-upstream_pid=$!
+isolated_workers=false
+if [ "$mode" = parity ]; then
+  export INTEGRATION_WORKERS=1
+else
+  case "$suite" in
+    management|accounting) export INTEGRATION_WORKERS=2; isolated_workers=true ;;
+    providers|extensions) export INTEGRATION_WORKERS=4; isolated_workers=true ;;
+  esac
+fi
+if [ "$isolated_workers" = false ]; then
+  setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+    .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
+  upstream_pid=$!
+fi
 if [ "$suite" = cost ]; then
   export INTEGRATION_WORKERS=8
 fi
@@ -180,17 +191,22 @@ start_proxy() {
     > "$results/$log_name" 2>&1 &
   launched_pid=$!
 }
-start_proxy 4000 proxy.log
-proxy_pid="$launched_pid"
-.venv/bin/python .circleci/scripts/wait_integration_services.py
-curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \
-  -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
-  -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json"
-if [ "$suite" = management ] || [ "$suite" = mcp ]; then
+if { [ "$suite" = management ] || [ "$suite" = mcp ]; } && [ "$isolated_workers" = true ]; then
   export INTEGRATION_PEER_URL=http://127.0.0.1:4001
-  start_proxy 4001 peer.log
-  peer_pid="$launched_pid"
+fi
+if [ "$isolated_workers" = false ]; then
+  start_proxy 4000 proxy.log
+  proxy_pid="$launched_pid"
   .venv/bin/python .circleci/scripts/wait_integration_services.py
+  curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \
+    -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
+    -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json"
+  if [ "$suite" = management ] || [ "$suite" = mcp ]; then
+    export INTEGRATION_PEER_URL=http://127.0.0.1:4001
+    start_proxy 4001 peer.log
+    peer_pid="$launched_pid"
+    .venv/bin/python .circleci/scripts/wait_integration_services.py
+  fi
 fi
 
 if [ "$suite" = providers ]; then
@@ -238,6 +254,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
   INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
   INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
+  MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
   .venv/bin/python tests/integration/run.py "$suite" --results "$results" \
   "${shard_arguments[@]}"
 

.circleci/scripts/run_migration_tests.py

@@ -10,7 +10,7 @@ from typing import Final
 from xml.etree import ElementTree
 
 SUITES: Final = {
-    "startup": (("test_startup.py",), 12),
+    "startup": (("test_startup.py",), 13),
     "recovery": (("test_recovery.py",), 15),
     "legacy": (("test_legacy.py", "test_pooling.py"), 11),
     "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),

.circleci/scripts/wait_integration_services.py

@@ -1,42 +1,31 @@
 import os
-import time
+from contextlib import ExitStack
 from typing import Final
 
 import httpx
+from integration._support.client import Gateway
+from integration._support.workers import wait_for_services
 from redis import Redis
 
 
 def main() -> None:
     primary: Final = os.environ["INTEGRATION_PROXY_URL"]
     peer: Final = os.environ.get("INTEGRATION_PEER_URL")
-    proxies: Final = (primary, peer) if peer else (primary,)
-    deadline: Final = time.monotonic() + 90
-    headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"}
-    with httpx.Client(trust_env=False, timeout=2) as client, Redis(
-        host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2
-    ) as cache:
-        while True:
-            try:
-                ready: Final = (
-                    client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200
-                    and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies)
-                )
-                if ready:
-                    for url in proxies:
-                        response: Final = client.get(f"{url}/cache/ping", headers=headers)
-                        response.raise_for_status()
-                        result: Final = response.json()
-                        assert result["status"] == "healthy", result
-                        assert result["cache_type"] == "redis", result
-                        assert result["ping_response"] is True, result
-                        assert result["set_cache_response"] == "success", result
-                    if cache.pubsub_numsub("litellm_proxy.auth_cache_invalidation")[0][1] >= len(proxies):
-                        return
-            except httpx.TransportError:
-                pass
-            if time.monotonic() >= deadline:
-                raise SystemExit("Integration services or auth-cache subscribers did not become ready")
-            time.sleep(0.2)
+    urls: Final = (primary, peer) if peer else (primary,)
+    upstream: Final = os.environ["INTEGRATION_UPSTREAM_URL"]
+    with ExitStack() as stack:
+        proxies: Final = tuple(
+            Gateway(
+                stack.enter_context(httpx.Client(base_url=url, trust_env=False, timeout=2)),
+                os.environ["INTEGRATION_MASTER_KEY"],
+                upstream,
+            )
+            for url in urls
+        )
+        cache: Final = stack.enter_context(
+            Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2)
+        )
+        wait_for_services(upstream, proxies, cache)
 
 
 if __name__ == "__main__":

.github/ci-coverage-allowlist.yml

@@ -23,18 +23,6 @@ test_paths:
       and that pull request gives it a new home
     paths:
       - tests/test_rust_python_harness.py
-  - reason: >-
-      Live-provider caching cases in tests/local_testing that remain outside CI. No CircleCI job
-      globs that directory anymore, and the legacy jobs that did always deselected these files.
-      The two Redis Cluster IAM authentication tests moved to tests/integration/sdk and remain
-      selected by node ID in redis_compat, where redis-server is installed.
-      The five keyless files now run in the caching-local shard, leaving live cases in these two.
-      Measured 2026-08-21 with no provider credentials: test_disk_cache_unit_tests.py needs
-      OPENAI_API_KEY for 2 of its 4 tests, and test_gcs_cache_unit_tests.py needs GCS credentials
-      for all 4. Keep these live cases outside PR CI rather than making a job red by construction
-    paths:
-      - tests/local_testing/test_disk_cache_unit_tests.py
-      - tests/local_testing/test_gcs_cache_unit_tests.py
   - reason: >-
       The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
       from a pull request; it needs a live gateway and provider credentials no PR job holds
@@ -60,48 +48,6 @@ test_paths:
       choice it informed is settled
     paths:
       - tests/code_coverage_tests/test_aio_http_image_conversion.py
-  - reason: >-
-      No job invokes this suite and its files mix pure transformation tests with ones driving live
-      vendor vector stores, so assigning them needs a per-file decision
-    paths:
-      - tests/vector_store_tests/rag/test_rag_bedrock.py
-      - tests/vector_store_tests/rag/test_rag_openai.py
-      - tests/vector_store_tests/rag/test_rag_s3_vectors.py
-      - tests/vector_store_tests/rag/test_rag_vertex_ai.py
-      - tests/vector_store_tests/test_azure_ai_vector_store.py
-      - tests/vector_store_tests/test_azure_vector_store.py
-      - tests/vector_store_tests/test_bedrock_vector_store.py
-      - tests/vector_store_tests/test_gemini_vector_store.py
-      - tests/vector_store_tests/test_milvus_vector_store.py
-      - tests/vector_store_tests/test_openai_vector_store.py
-      - tests/vector_store_tests/test_ragflow_vector_store.py
-      - tests/vector_store_tests/test_s3_vectors_vector_store.py
-      - tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py
-      - tests/vector_store_tests/test_vertex_ai_vector_store.py
-  - reason: >-
-      Throughput and memory-growth measurements whose runtime and variance make them unsuitable for
-      a per-pull-request job
-    paths:
-      - tests/load_tests/test_datadog_load_test.py
-      - tests/load_tests/test_langsmith_load_test.py
-      - tests/load_tests/test_linear_memory_growth.py
-      - tests/load_tests/test_memory_usage.py
-      - tests/load_tests/test_otel_load_test.py
-      - tests/load_tests/test_vertex_embeddings_load_test.py
-      - tests/load_tests/test_vertex_load_tests.py
-  - reason: >-
-      Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
-      locally rather than in pull-request jobs
-    paths:
-      - tests/load_tests/test_granian_admission_saturation.py
-  - reason: >-
-      A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
-      localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
-      pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v
-      that this census could not see; the glob now excludes them structurally and this entry
-      is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture
-    paths:
-      - tests/agent_tests/local_only_agent_tests
   - reason: >-
       Third-party integration tests that skip themselves without OCI configuration or sandbox
       credentials, neither of which a pull request job holds

.github/scripts/assert_ci_coverage.py

@@ -37,7 +37,6 @@ GLOB_CHARS = frozenset("*?")
 # tests has to be named by some shard or it runs nowhere. A child listed here is
 # itself decomposed one level deeper and is checked through its own entry.
 SHARDED_ROOTS: tuple[str, ...] = (
-    "tests/test_litellm",
     "tests/unit",
     "tests/unit/enterprise",
     "tests/unit/enterprise/enterprise_callbacks",

.github/workflows/deploy-dev-on-main.yml

@@ -0,0 +1,30 @@
+name: Deploy main to gateway-dev
+
+on:
+  push:
+    branches: [main]
+
+permissions: {}
+
+concurrency:
+  group: deploy-dev-dispatch
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
+        id: token
+        with:
+          app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
+          private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
+          owner: BerriAI
+          repositories: litellm-ops
+          permission-actions: write
+      - name: Dispatch the litellm-ops dev deploy
+        env:
+          GH_TOKEN: ${{ steps.token.outputs.token }}
+        run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

.github/workflows/image-scan.yml

@@ -27,7 +27,7 @@ on:
       - backend/routes/allowlist.py
       - pyproject.toml
       - litellm-proxy-extras/**
-      - tests/proxy_migration_tests/**
+      - tests/integration/database/image/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - ui/Dockerfile
@@ -92,7 +92,7 @@ jobs:
           LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
 
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
@@ -145,7 +145,7 @@ jobs:
           LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
 
 
   migrations-image:
@@ -177,7 +177,7 @@ jobs:
           LITELLM_MIGRATION_SCRIPT: /app/run.py
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v
 
   gateway-image:
     name: gateway-image
@@ -208,7 +208,7 @@ jobs:
           LITELLM_IMAGE_COMPONENT: gateway
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
 
   ui-image:
     name: ui-image
@@ -237,7 +237,7 @@ jobs:
           LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v
 
   backend-image:
     name: backend-image
@@ -272,4 +272,4 @@ jobs:
           LITELLM_IMAGE_COMPONENT: backend
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
+          python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v

.github/workflows/test-linting.yml

@@ -22,6 +22,16 @@ jobs:
       actions: read
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    services:
+      redis:
+        image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
@@ -189,8 +199,9 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         env:
           LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
+          E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0
         run: |
-          if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
+          if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' tests/code_coverage_tests/test_provider_replay_harness.py tests/code_coverage_tests/test_provider_cache.py pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
             echo "No changed e2e harness files; skipping."
             exit 0
           fi
@@ -198,6 +209,9 @@ jobs:
           CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
           tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
           PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
+          uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
+            tests/code_coverage_tests/test_provider_replay_harness.py \
+            tests/code_coverage_tests/test_provider_cache.py
 
       - name: Check for circular imports
         if: steps.changes.outputs.decision != 'skip'

.github/workflows/test-merge-smoke.yml

@@ -234,7 +234,30 @@ jobs:
           exit 1
 
       - name: Run the startup tests
-        run: .venv/bin/python -m pytest --tb=short -vv tests/basic_proxy_startup_tests
+        shell: bash
+        run: |
+          set -euo pipefail
+          base_url=http://localhost:4000
+          readiness_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' "${base_url}/health/readiness")"
+          readiness_status="${readiness_response##*$'\n'}"
+          readiness_body="${readiness_response%$'\n'*}"
+          [[ "${readiness_status}" == "200" ]]
+          jq -e '.status == "healthy" or .status == "connected"' <<< "${readiness_body}" > /dev/null
+          liveness_status="$(curl -sS --fail-with-body --output /dev/null --write-out '%{http_code}' "${base_url}/health/liveness")"
+          [[ "${liveness_status}" == "200" ]]
+          completion_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' \
+            --header "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
+            --header 'Content-Type: application/json' \
+            --data '{"model":"gpt-4","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello!"}]}' \
+            "${base_url}/chat/completions")"
+          completion_status="${completion_response##*$'\n'}"
+          completion_body="${completion_response%$'\n'*}"
+          [[ "${completion_status}" == "200" ]]
+          jq -e '
+            (.choices | length) == 1 and
+            .choices[0].message.role == "assistant" and
+            .choices[0].message.content == "Hello! This is a mock response from the fake OpenAI endpoint."
+          ' <<< "${completion_body}" > /dev/null
 
       - name: Show the proxy logs
         if: always()

.github/workflows/test-rust.yml

@@ -18,7 +18,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
       - "pyproject.toml"
@@ -49,7 +48,6 @@ on:
       - "litellm/ocr/**"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
-      - "tests/test_litellm/conftest.py"
       - "Makefile"
       - ".cargo/**"
       - "pyproject.toml"

.github/workflows/test-unit.yml

@@ -174,8 +174,6 @@ jobs:
 
           - shard: integrations
             test-path: |-
-              tests/test_litellm/integrations
-              tests/test_litellm/tracing
               tests/unit/integrations
             python-version: "3.12"
             workers: 4
@@ -292,6 +290,7 @@ jobs:
               tests/unit/llms/manus
               tests/unit/llms/meta_llama
               tests/unit/llms/microsoft_365_copilot
+              tests/unit/llms/milvus
               tests/unit/llms/minimax
               tests/unit/llms/mistral
               tests/unit/llms/modelscope
@@ -376,7 +375,6 @@ jobs:
 
           - shard: root
             test-path: >-
-              tests/test_litellm/test_*.py
               tests/unit/test_*.py
             python-version: "3.12"
             workers: 4
@@ -395,16 +393,6 @@ jobs:
             job-timeout-minutes: 60
             dist: loadscope
 
-          - shard: router-unit-tests
-            test-path: >-
-              tests/router_unit_tests
-            python-version: "3.12"
-            workers: 0
-            reruns: 0
-            timeout-minutes: 10
-            job-timeout-minutes: 60
-            dist: loadscope
-
           - shard: endpoints
             test-path: >-
               tests/unit/a2a_protocol
@@ -775,7 +763,7 @@ jobs:
 
           - shard: proxy-server-core
             test-path: |-
-              tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
+              tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py
               tests/unit/proxy/test__lazy_features.py
               tests/unit/proxy/test_aproxy_startup.py
               tests/unit/proxy/test_proxy_server.py
@@ -816,7 +804,7 @@ jobs:
 
           - shard: custom-logging
             test-path: |-
-              tests/proxy_unit_tests/test_proxy_custom_logger.py
+              tests/unit/proxy/test_proxy_server_custom_logger.py
               tests/unit/proxy/test_custom_callback_input.py
               tests/unit/proxy/test_custom_logger_s3_gcs.py
             python-version: "3.12"
@@ -880,7 +868,7 @@ jobs:
 
           - shard: endpoints-and-responses
             test-path: |-
-              tests/proxy_unit_tests/test_proxy_exception_mapping.py
+              tests/unit/proxy/test_proxy_server_exception_mapping.py
               tests/unit/proxy/lens
               tests/unit/proxy/auth/test_models_fallback_endpoint.py
               tests/unit/proxy/common_utils/test_check_batch_cost.py

.greptile/files.json

@@ -28,6 +28,13 @@
         "litellm/rust_bridge/**"
       ]
     },
+    {
+      "path": "litellm/rust_bridge/host/AGENTS.md",
+      "description": "Conventions for code under litellm/rust_bridge/host/",
+      "scope": [
+        "litellm/rust_bridge/host/**"
+      ]
+    },
     {
       "path": "litellm/tracing/AGENTS.md",
       "description": "Conventions for code under litellm/tracing/",
@@ -266,6 +273,13 @@
         "litellm-rust/crates/llms/src/bedrock/chat/**"
       ]
     },
+    {
+      "path": "litellm-rust/crates/llms/src/bedrock/claude_platform/messages/AGENTS.md",
+      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/claude_platform/messages/",
+      "scope": [
+        "litellm-rust/crates/llms/src/bedrock/claude_platform/messages/**"
+      ]
+    },
     {
       "path": "litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md",
       "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/messages/",
@@ -273,6 +287,13 @@
         "litellm-rust/crates/llms/src/bedrock/messages/**"
       ]
     },
+    {
+      "path": "litellm-rust/crates/llms/src/bedrock_mantle/messages/AGENTS.md",
+      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock_mantle/messages/",
+      "scope": [
+        "litellm-rust/crates/llms/src/bedrock_mantle/messages/**"
+      ]
+    },
     {
       "path": "litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md",
       "description": "Conventions for code under litellm-rust/crates/llms/src/cohere/ocr/",
@@ -287,6 +308,13 @@
         "litellm-rust/crates/llms/src/deepseek/messages/**"
       ]
     },
+    {
+      "path": "litellm-rust/crates/llms/src/minimax/messages/AGENTS.md",
+      "description": "Conventions for code under litellm-rust/crates/llms/src/minimax/messages/",
+      "scope": [
+        "litellm-rust/crates/llms/src/minimax/messages/**"
+      ]
+    },
     {
       "path": "litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md",
       "description": "Conventions for code under litellm-rust/crates/llms/src/mistral/ocr/",
@@ -308,6 +336,13 @@
         "litellm-rust/crates/llms/src/openai_like/chat/**"
       ]
     },
+    {
+      "path": "litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md",
+      "description": "Conventions for code under litellm-rust/crates/llms/src/openrouter/messages/",
+      "scope": [
+        "litellm-rust/crates/llms/src/openrouter/messages/**"
+      ]
+    },
     {
       "path": "litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md",
       "description": "Conventions for code under litellm-rust/crates/llms/src/reducto/ocr/",
@@ -315,6 +350,13 @@
         "litellm-rust/crates/llms/src/reducto/ocr/**"
       ]
     },
+    {
+      "path": "litellm-rust/crates/llms/src/tencent/messages/AGENTS.md",
+      "description": "Conventions for code under litellm-rust/crates/llms/src/tencent/messages/",
+      "scope": [
+        "litellm-rust/crates/llms/src/tencent/messages/**"
+      ]
+    },
     {
       "path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md",
       "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/",

Makefile

@@ -39,7 +39,7 @@ help:
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests (tests/unit and tests/test_litellm)"
+	@echo "  make test-unit          - Run unit tests (tests/unit)"
 	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
 	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
 	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
@@ -300,7 +300,7 @@ test: install-test-deps
 	$(UV_RUN) pytest tests/
 
 test-unit: install-test-deps
-	$(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4
+	$(UV_RUN) pytest tests/unit -x -vv -n 4
 
 # Matrix test targets (matching CI workflow groups)
 test-unit-llms: install-test-deps
@@ -328,7 +328,7 @@ test-unit-other: install-test-deps
 	$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
-	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/test_*.py --tb=short -vv -n 4 --durations=20
 
 # Proxy unit tests (tests/unit/proxy split alphabetically)
 test-proxy-unit-a: install-test-deps
@@ -338,7 +338,7 @@ test-proxy-unit-b: install-test-deps
 	$(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py tests/unit/skills --tb=short -vv -n 2 --durations=20
 
 test-integration: install-test-deps
-	$(UV_RUN) pytest tests/ -k "not test_litellm"
+	$(UV_RUN) pytest tests/
 
 test-unit-helm: install-helm-unittest
 	helm unittest -f 'tests/*.yaml' helm/litellm-helm

codecov.yaml

@@ -71,6 +71,8 @@ flag_management:
       carryforward: false
     - name: lens-python-310
       carryforward: false
+    - name: lens-postgres
+      carryforward: false
     - name: proxy-db-key-generation
       carryforward: false
     - name: proxy-db-auth-checks

litellm-rust/crates/callbacks-legacy-python/src/python.rs

@@ -1,7 +1,7 @@
 use pyo3::prelude::*;
 use strum::{IntoStaticStr, VariantArray};
 
-const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
+const MODULE: &str = "litellm.rust_bridge.host.callbacks_legacy_python";
 
 /// Every litellm Python internal the native call still borrows, grouped by the subsystem it
 /// belongs to. Rust drives the call; these exist only so behaviour that Python owns today
@@ -9,7 +9,7 @@ const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
 /// A group is deleted once Rust owns that subsystem, so this enum only shrinks. Calling a
 /// user's own callback is not borrowing and does not belong here.
 ///
-/// `litellm/rust_bridge/callbacks_legacy_python.py` is the only Python module behind it, and
+/// `litellm/rust_bridge/host/callbacks_legacy_python.py` is the only Python module behind it, and
 /// `python_contract.json` pins each function's parameters on both sides.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub(crate) enum LegacyPython {

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyTuple};
 use crate::{LegacyLogging, PublicCall};
 
 /// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
-/// `tests/unit/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
+/// `tests/unit/rust_bridge/host/test_callbacks_legacy_python.py` pins this file to the Python
 /// signatures, and [`namespace`] binds every fake call against it.
 pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
 
@@ -23,10 +23,10 @@ import sys
 import traceback
 import types
 
-for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy_python'):
+for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.callbacks_legacy_python'):
     sys.modules.setdefault(name, types.ModuleType(name))
 
-legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
+legacy = sys.modules['litellm.rust_bridge.host.callbacks_legacy_python']
 CONTRACT = json.loads(python_contract)
 
 

litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs

@@ -16,18 +16,26 @@ pub enum LlmProviders {
     AzureAi,
     #[strum(serialize = "bedrock")]
     Bedrock,
+    #[strum(serialize = "bedrock_mantle")]
+    BedrockMantle,
     #[strum(serialize = "cohere")]
     Cohere,
     #[strum(serialize = "deepseek")]
     Deepseek,
+    #[strum(serialize = "minimax")]
+    Minimax,
     #[strum(serialize = "mistral")]
     Mistral,
     #[strum(serialize = "openai")]
     Openai,
     #[strum(serialize = "openai_like")]
     OpenaiLike,
+    #[strum(serialize = "openrouter")]
+    Openrouter,
     #[strum(serialize = "reducto")]
     Reducto,
+    #[strum(serialize = "tencent")]
+    Tencent,
     #[strum(serialize = "vertex_ai")]
     VertexAi,
 }

litellm-rust/crates/gateway-inference/src/messages.rs

@@ -19,7 +19,12 @@ use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
 
 /// Client headers Python forwards to Anthropic-speaking providers on every call.
 const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
-const ANTHROPIC_API_HEADER_PROVIDERS: &str = "anthropic,bedrock,bedrock_mantle,vertex_ai";
+const OPENROUTER_API_HEADERS: [&str; 2] = [
+    "x-openrouter-metadata",
+    "x-openrouter-experimental-metadata",
+];
+const ANTHROPIC_API_HEADER_PROVIDERS: &str =
+    "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
 
 pub async fn create(
     State(gateway): State<Arc<Gateway>>,
@@ -88,17 +93,28 @@ fn project(
 }
 
 fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
-    let extra_headers: Map<String, Value> = ANTHROPIC_API_HEADERS
+    let scopes = [
+        (
+            ANTHROPIC_API_HEADER_PROVIDERS,
+            ANTHROPIC_API_HEADERS.as_slice(),
+        ),
+        ("openrouter", OPENROUTER_API_HEADERS.as_slice()),
+    ];
+    let forwarded: Vec<ProviderSpecificHeader> = scopes
         .into_iter()
-        .filter_map(|name| {
-            let value = headers.get(name)?.to_str().ok()?;
-            Some((name.to_owned(), Value::from(value)))
+        .filter_map(|(provider, names)| {
+            let extra_headers: Map<String, Value> = names
+                .iter()
+                .filter_map(|name| {
+                    let value = headers.get(*name)?.to_str().ok()?;
+                    Some(((*name).to_owned(), Value::from(value)))
+                })
+                .collect();
+            (!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
+                custom_llm_provider: provider.into(),
+                extra_headers,
+            })
         })
         .collect();
-    (!extra_headers.is_empty()).then(|| {
-        ProviderSpecificHeaders::One(ProviderSpecificHeader {
-            custom_llm_provider: ANTHROPIC_API_HEADER_PROVIDERS.into(),
-            extra_headers,
-        })
-    })
+    (!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
 }

litellm-rust/crates/gateway-inference/tests/messages.rs

@@ -192,3 +192,52 @@ async fn model_permissions_prevent_provider_calls(#[case] path: &str) {
     assert_eq!(response.status(), 403);
     assert!(upstream.received_requests().await.unwrap().is_empty());
 }
+
+#[rstest]
+#[case::completion(false)]
+#[case::streaming(true)]
+#[tokio::test]
+async fn openrouter_receives_native_messages_headers_and_its_deployment_bearer(
+    #[case] streaming: bool,
+) {
+    let upstream = MockServer::start().await;
+    let message = json!({"id": "msg_test", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": null, "usage": {"input_tokens": 1, "output_tokens": 1, "cost": 0.001, "cost_details": "unexpected"}, "openrouter_metadata": {"provider": "test-provider"}});
+    let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":0.001}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
+    let template = if streaming {
+        ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")
+    } else {
+        ResponseTemplate::new(200).set_body_json(message.clone())
+    };
+    Mock::given(method("POST"))
+        .and(path("/api/v1/messages"))
+        .and(header("authorization", "Bearer test-key"))
+        .and(header("anthropic-beta", "test-feature"))
+        .and(header("anthropic-version", "test-version"))
+        .and(header("x-openrouter-metadata", "enabled"))
+        .and(body_json(json!({"model": "anthropic/test-model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}})))
+        .respond_with(template).expect(1).mount(&upstream).await;
+    let request = Request::post("/v1/messages")
+        .header("content-type", "application/json")
+        .header("anthropic-beta", "test-feature")
+        .header("anthropic-version", "test-version")
+        .header("x-openrouter-metadata", "enabled")
+        .header("x-api-key", "proxy-key")
+        .header("authorization", "Bearer proxy-key")
+        .body(Body::from(json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}}).to_string())).unwrap();
+    let response = support::app(
+        "openrouter/anthropic/test-model",
+        &format!("{}/api/v1", upstream.uri()),
+    )
+    .oneshot(request)
+    .await
+    .unwrap();
+    assert_eq!(response.status(), 200);
+    if streaming {
+        assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), sse);
+    } else {
+        assert_eq!(support::json(response).await, message);
+    }
+    let requests = upstream.received_requests().await.unwrap();
+    assert_eq!(requests.len(), 1);
+    assert!(!requests[0].headers.contains_key("x-api-key"));
+}

litellm-rust/crates/gateway-mcp/AGENTS.md

@@ -24,7 +24,7 @@ From the repository root, this loopback-only example runs the existing Python MC
 
 ```sh
 cargo run --manifest-path litellm-rust/Cargo.toml -p litellm-gateway-mcp --example stdio_proxy -- \
-  .venv/bin/python -c 'import runpy; runpy.run_path("tests/mcp_tests/mcp_e2e_upstream_server.py")["mcp"].run(transport="stdio")'
+  .venv/bin/python -c 'import runpy; runpy.run_path("tests/integration/_support/mcp_e2e_upstream_server.py")["mcp"].run(transport="stdio")'
 ```
 
 ```sh

litellm-rust/crates/gateway/README.md

@@ -7,7 +7,7 @@ The binary reads YAML from `LITELLM_CONFIG` and mounts configured MCP servers al
 From the repository root, start the Python fixture in one terminal:
 
 ```sh
-MCP_HOST=127.0.0.1 MCP_PORT=8090 .venv/bin/python tests/mcp_tests/mcp_e2e_upstream_server.py
+MCP_HOST=127.0.0.1 MCP_PORT=8090 .venv/bin/python tests/integration/_support/mcp_e2e_upstream_server.py
 ```
 
 Start the gateway in another terminal with the example configuration:

litellm-rust/crates/http/src/request.rs

@@ -138,7 +138,9 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
         }
         let value = value.trim();
         value.len() > 7
-            && value[..7].eq_ignore_ascii_case("bearer ")
+            && value
+                .get(..7)
+                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("bearer "))
             && !value[7..].trim().is_empty()
     })
 }
@@ -279,31 +281,19 @@ mod tests {
         assert!(!has_header(&headers, "authorization"));
     }
 
-    #[test]
-    fn bearer_detection_requires_a_non_empty_token() {
-        assert!(has_bearer_auth(&[(
-            "Authorization".to_string(),
-            "Bearer abc".to_string()
-        )]));
-        assert!(has_bearer_auth(&[(
-            "authorization".to_string(),
-            "bearer abc".to_string()
-        )]));
-        assert!(!has_bearer_auth(&[(
-            "Authorization".to_string(),
-            "Bearer    ".to_string()
-        )]));
-        assert!(!has_bearer_auth(&[(
-            "authorization".to_string(),
-            String::new()
-        )]));
-        assert!(!has_bearer_auth(&[(
-            "Authorization".to_string(),
-            "Basic abc".to_string()
-        )]));
-        assert!(!has_bearer_auth(&[(
-            "x-api-key".to_string(),
-            "abc".to_string()
-        )]));
+    #[rstest::rstest]
+    #[case::bearer("Authorization", "Bearer abc", true)]
+    #[case::lowercase("authorization", "bearer abc", true)]
+    #[case::blank_token("Authorization", "Bearer    ", false)]
+    #[case::empty("authorization", "", false)]
+    #[case::basic("Authorization", "Basic abc", false)]
+    #[case::different_header("x-api-key", "abc", false)]
+    #[case::non_ascii("authorization", "invali\u{00e9}auth", false)]
+    fn bearer_detection_requires_a_non_empty_token(
+        #[case] name: &str,
+        #[case] value: &str,
+        #[case] expected: bool,
+    ) {
+        assert_eq!(has_bearer_auth(&[(name.into(), value.into())]), expected);
     }
 }

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -1,14 +1,20 @@
 use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 use litellm_http::request::string_headers as shared_string_headers;
 pub(super) use litellm_http::request::truncate_error_body;
+use litellm_llms::tencent::messages::transformation::TENCENT_MESSAGES_CONFIG;
 use litellm_llms::{
     anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG,
     azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG,
     base_llm::messages::transformation::BaseMessagesConfig,
     bedrock::messages::invoke_transformations::anthropic_claude3_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
     deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
+    minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG,
     vertex_ai::messages::transformation::VERTEX_ANTHROPIC_MESSAGES_CONFIG,
 };
+use litellm_llms::{
+    bedrock::claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG,
+    bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG,
+};
 use serde_json::{Map, Value};
 
 use super::Error;
@@ -34,14 +40,24 @@ impl MessagesProvider {
 /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
 /// partner models on this route.
 pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<MessagesProvider> {
+    use litellm_llms::openrouter::messages::transformation::OPENROUTER_MESSAGES_CONFIG;
+
     let config: &'static dyn BaseMessagesConfig = match provider {
+        LlmProviders::Tencent => &TENCENT_MESSAGES_CONFIG,
         LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
         LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
+        LlmProviders::Bedrock if model.starts_with("claude_platform/") => {
+            &BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+        }
+        LlmProviders::Bedrock if model.starts_with("mantle/") => &BEDROCK_MANTLE_MESSAGES_CONFIG,
         LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
+        LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG,
         LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
+        LlmProviders::Minimax => &MINIMAX_MESSAGES_CONFIG,
         LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => {
             &VERTEX_ANTHROPIC_MESSAGES_CONFIG
         }
+        LlmProviders::Openrouter => &OPENROUTER_MESSAGES_CONFIG,
         _ => return None,
     };
     Some(MessagesProvider { provider, config })
@@ -64,10 +80,12 @@ mod tests {
     use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 
     #[rstest]
+    #[case::tencent("tencent")]
     #[case::anthropic("anthropic")]
     #[case::azure_ai("azure_ai")]
     #[case::bedrock("bedrock")]
     #[case::deepseek("deepseek")]
+    #[case::minimax("minimax")]
     #[case::vertex_ai("vertex_ai")]
     fn provider_keeps_its_python_name(#[case] name: &str) {
         let provider =

litellm-rust/crates/inference-messages/src/prepare.rs

@@ -76,6 +76,10 @@ fn prepare_provider_request(
         shaping,
         ..
     } = call;
+    let connection_params = litellm_router_types::LitellmParams {
+        api_base: api_base.clone().or(litellm_params.api_base.clone()),
+        ..litellm_params
+    };
     let config = provider.config();
     let env_lookup = |key: &str| secrets.get(key);
 
@@ -95,25 +99,22 @@ fn prepare_provider_request(
     let forwarded = string_headers(Some(
         extra_headers.into_iter().flatten().chain(scoped).collect(),
     ))?;
-    let validated = config.validate_environment(
+    let forwarded = config.request_headers(
+        with_default_headers(forwarded, config.default_headers()),
+        &transformed,
+    );
+    let environment = config.validate_environment(
         forwarded,
         api_key.as_deref(),
         &transformed.model,
-        &litellm_params,
+        &connection_params,
         &env_lookup,
     )?;
-    let environment = ValidatedEnvironment {
-        headers: config.request_headers(
-            with_default_headers(validated.headers, config.default_headers()),
-            &transformed,
-        ),
-        auth: validated.auth,
-    };
 
     let url = config.get_complete_url(
         api_base.as_deref(),
         &transformed.model,
-        &litellm_params,
+        &connection_params,
         transformed.params.stream == Some(true),
         &env_lookup,
     )?;
@@ -475,4 +476,44 @@ mod tests {
             ))
         );
     }
+    #[rstest]
+    #[case::mantle(
+        "bedrock_mantle/claude-test",
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "/anthropic/v1/messages",
+        "bedrock"
+    )]
+    #[case::platform(
+        "bedrock/claude_platform/claude-test",
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "/v1/messages",
+        "aws-external-anthropic"
+    )]
+    fn explicit_endpoint_region_is_used_by_the_signer(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] base: &str,
+        #[case] path: &str,
+        #[case] service: &str,
+    ) {
+        let prepared = prepare_with_secrets(MessagesCall {
+            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
+            api_key: None,
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        }, &|name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
+            _ => None,
+        }).unwrap();
+        assert_eq!(prepared.url, format!("{base}{path}"));
+        assert!(
+            matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
+        );
+    }
 }

litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs

@@ -0,0 +1,323 @@
+use std::{sync::Arc, time::Duration};
+
+use futures_util::TryStreamExt;
+use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping};
+use litellm_inference_testing::{RecordingSecrets, no_secrets};
+use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use rstest::{fixture, rstest};
+use serde_json::{Value, json};
+use wiremock::ResponseTemplate;
+
+#[path = "../support/mod.rs"]
+mod support;
+use support::*;
+
+const MODEL: &str = "claude-test";
+
+#[fixture]
+fn call() -> MessagesCall {
+    MessagesCall {
+        body: serde_json::from_value(json!({"model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})).unwrap(),
+        api_key: None,
+        api_base: None,
+        custom_llm_provider: None,
+        litellm_params: Default::default(),
+        extra_headers: None,
+        provider_specific_header: None,
+        timeout: Some(Duration::from_secs(5)),
+        shaping: MessagesShaping::default(),
+    }
+}
+
+fn with_fields(call: MessagesCall, fields: Value) -> MessagesCall {
+    let body = serde_json::to_value(call.body).unwrap();
+    MessagesCall {
+        body: serde_json::from_value(Value::Object(
+            body.as_object()
+                .unwrap()
+                .clone()
+                .into_iter()
+                .chain(fields.as_object().unwrap().clone())
+                .collect(),
+        ))
+        .unwrap(),
+        ..call
+    }
+}
+
+fn with_model(call: MessagesCall, model: &str) -> MessagesCall {
+    MessagesCall {
+        body: MessagesRequest {
+            model: model.into(),
+            ..call.body
+        },
+        ..call
+    }
+}
+
+fn message_response() -> ResponseTemplate {
+    json_response(
+        json!({"id": "msg_1", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}], "model": MODEL, "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 2}}),
+    )
+}
+
+async fn run_with(
+    secrets: Arc<RecordingSecrets>,
+    call: MessagesCall,
+) -> Result<MessagesCallResponse, Error> {
+    messages_route(secrets).execute(call, &(), None).await
+}
+
+async fn run(call: MessagesCall) -> Result<MessagesCallResponse, Error> {
+    run_with(Arc::new(RecordingSecrets::empty()), call).await
+}
+
+async fn run_message(call: MessagesCall) -> MessagesResponse {
+    match run(call).await.unwrap() {
+        MessagesCallResponse::Complete(message) => *message,
+        MessagesCallResponse::Stream { .. } => panic!("expected a completed message"),
+    }
+}
+
+#[rstest]
+#[case::platform(
+    "bedrock/claude_platform/claude-test",
+    "/v1/messages",
+    "x-api-key",
+    "key"
+)]
+#[case::mantle(
+    "bedrock_mantle/anthropic.claude-test",
+    "/anthropic/v1/messages",
+    "authorization",
+    "Bearer key"
+)]
+#[case::mantle_alias(
+    "bedrock/mantle/anthropic.claude-test",
+    "/anthropic/v1/messages",
+    "authorization",
+    "Bearer key"
+)]
+#[tokio::test]
+async fn prefixes_select_native_messages_and_strip_connection_fields(
+    call: MessagesCall,
+    #[case] model: &str,
+    #[case] path: &str,
+    #[case] header: &str,
+    #[case] credential: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+    let message = run_message(MessagesCall {
+        api_key: Some("key".into()),
+        api_base: Some(upstream.uri()),
+        custom_llm_provider: None,
+        ..with_fields(
+            call,
+            json!({"model": model, "workspace_id": "request-workspace"}),
+        )
+    })
+    .await;
+    assert_eq!(message.model, MODEL);
+    let request = only_request(&upstream).await;
+    assert_eq!(request.url.path(), path);
+    assert_eq!(request.header(header), Some(credential));
+    assert_eq!(request.json()["model"], model.rsplit('/').next().unwrap());
+    if path == "/v1/messages" {
+        assert_eq!(
+            request.header("anthropic-workspace-id"),
+            Some("request-workspace")
+        );
+        assert_eq!(request.json().get("workspace_id"), None);
+    }
+}
+
+#[rstest]
+#[case::platform(
+    "bedrock/claude_platform/claude-test",
+    "ANTHROPIC_AWS_API_KEY",
+    "ANTHROPIC_AWS_BASE_URL",
+    "/v1/messages",
+    "x-api-key",
+    "managed"
+)]
+#[case::mantle(
+    "bedrock_mantle/claude-test",
+    "AWS_BEARER_TOKEN_BEDROCK",
+    "BEDROCK_MANTLE_API_BASE",
+    "/anthropic/v1/messages",
+    "authorization",
+    "Bearer managed"
+)]
+#[tokio::test]
+async fn managed_keys_bases_and_workspaces_reach_the_upstream(
+    call: MessagesCall,
+    #[case] model: &str,
+    #[case] key_name: &str,
+    #[case] base_name: &str,
+    #[case] path: &str,
+    #[case] header: &str,
+    #[case] credential: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+    let base = upstream.uri();
+    let secrets = Arc::new(RecordingSecrets::new([
+        (key_name, "managed"),
+        (base_name, base.as_str()),
+        ("ANTHROPIC_AWS_WORKSPACE_ID", "managed-workspace"),
+    ]));
+    let output = run_with(
+        secrets,
+        MessagesCall {
+            custom_llm_provider: None,
+            ..with_model(call, model)
+        },
+    )
+    .await
+    .unwrap();
+    assert!(matches!(output, MessagesCallResponse::Complete(_)));
+    let request = only_request(&upstream).await;
+    assert_eq!(request.url.path(), path);
+    assert_eq!(request.header(header), Some(credential));
+    if path == "/v1/messages" {
+        assert_eq!(
+            request.header("anthropic-workspace-id"),
+            Some("managed-workspace")
+        );
+    }
+}
+
+#[rstest]
+#[case::platform("bedrock/claude_platform/claude-test", "aws-external-anthropic")]
+#[case::mantle("bedrock_mantle/claude-test", "bedrock")]
+#[tokio::test]
+async fn managed_aws_credentials_sign_the_exact_native_request(
+    call: MessagesCall,
+    #[case] model: &str,
+    #[case] service: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+    let secrets = Arc::new(RecordingSecrets::new([
+        ("AWS_ACCESS_KEY_ID", "test-access"),
+        ("AWS_SECRET_ACCESS_KEY", "test-secret"),
+        ("AWS_SESSION_TOKEN", "test-session"),
+        ("AWS_REGION_NAME", "us-west-2"),
+        ("ANTHROPIC_AWS_WORKSPACE_ID", "managed-workspace"),
+    ]));
+    run_with(
+        secrets,
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            custom_llm_provider: None,
+            ..with_model(call, model)
+        },
+    )
+    .await
+    .unwrap();
+    let request = only_request(&upstream).await;
+    let auth = request.header("authorization").unwrap();
+    assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/"));
+    assert!(auth.contains(&format!("/us-west-2/{service}/aws4_request")));
+    assert_eq!(request.header("x-amz-security-token"), Some("test-session"));
+    assert_eq!(request.header("x-api-key"), None);
+    assert_eq!(request.json()["model"], "claude-test");
+}
+
+#[rstest]
+#[tokio::test]
+async fn private_mantle_endpoint_is_used_and_signed(call: MessagesCall) {
+    let upstream = upstream([message_response()]).await;
+    let params = litellm_router_types::LitellmParams {
+        aws: litellm_auth::AwsParams {
+            aws_bedrock_runtime_endpoint: Some(upstream.uri()),
+            aws_access_key_id: Some("private-access".into()),
+            aws_secret_access_key: Some("private-secret".into()),
+            aws_region_name: Some("us-west-2".into()),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    run(MessagesCall {
+        custom_llm_provider: None,
+        litellm_params: params,
+        ..with_model(call, "bedrock/mantle/claude-test")
+    })
+    .await
+    .unwrap();
+    let request = only_request(&upstream).await;
+    assert_eq!(request.url.path(), "/anthropic/v1/messages");
+    assert!(
+        request
+            .header("authorization")
+            .unwrap()
+            .contains("/us-west-2/bedrock/aws4_request")
+    );
+}
+
+#[rstest]
+#[case::platform("bedrock/claude_platform/claude-test")]
+#[case::mantle("bedrock_mantle/claude-test")]
+#[tokio::test]
+async fn native_streams_relay_every_sse_byte(call: MessagesCall, #[case] model: &str) {
+    let sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
+    let upstream =
+        upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
+    let response = messages_route(no_secrets())
+        .execute(
+            MessagesCall {
+                api_key: Some("key".into()),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: None,
+                ..with_fields(
+                    call,
+                    json!({"model": model, "stream": true, "workspace_id": "workspace"}),
+                )
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+    let MessagesCallResponse::Stream { chunks, .. } = response else {
+        panic!("expected SSE");
+    };
+    assert_eq!(
+        chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
+        sse.as_bytes()
+    );
+    assert_eq!(only_request(&upstream).await.json()["stream"], true);
+}
+
+#[rstest]
+#[case::missing(false)]
+#[case::configured(true)]
+#[tokio::test]
+async fn deployment_workspace_is_required_and_sent_as_a_header(
+    call: MessagesCall,
+    #[case] configured: bool,
+) {
+    let upstream = upstream([message_response()]).await;
+    let params = if configured {
+        serde_json::from_value(json!({"model": "bedrock/claude_platform/claude-test", "workspace_id": "deployment-workspace"})).unwrap()
+    } else {
+        Default::default()
+    };
+    let result = run(MessagesCall {
+        api_key: Some("key".into()),
+        api_base: Some(upstream.uri()),
+        litellm_params: params,
+        ..with_model(call, "bedrock/claude_platform/claude-test")
+    })
+    .await;
+    if !configured {
+        assert!(matches!(result, Err(Error::Auth(_))));
+        assert!(received(&upstream).await.is_empty());
+        return;
+    }
+    assert!(matches!(result.unwrap(), MessagesCallResponse::Complete(_)));
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.header("anthropic-workspace-id"),
+        Some("deployment-workspace")
+    );
+    assert_eq!(sent.json().get("workspace_id"), None);
+}

litellm-rust/crates/inference-messages/tests/messages/main.rs

@@ -24,6 +24,7 @@ mod request;
 mod response;
 mod secrets;
 mod stream;
+mod tencent;
 
 const MODEL: &str = "claude-sonnet-4-5";
 

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -11,6 +11,9 @@ use super::*;
 
 #[rstest]
 #[case::anthropic_key("anthropic", Some("sk-ant"), &[], ("x-api-key", "sk-ant"), &["authorization"])]
+#[case::minimax_key("minimax", Some("sk-test"), &[], ("x-api-key", "sk-test"), &["authorization"])]
+#[case::minimax_forwarded_key("minimax", Some("deployment"), &[("X-Api-Key", "caller")], ("x-api-key", "caller"), &["authorization"])]
+#[case::minimax_forwarded_bearer("minimax", None, &[("Authorization", "Bearer caller")], ("authorization", "Bearer caller"), &["x-api-key"])]
 #[case::azure_key("azure_ai", Some("sk-azure"), &[], ("x-api-key", "sk-azure"), &["authorization"])]
 #[case::deepseek_key("deepseek", Some("sk-deepseek"), &[], ("x-api-key", "sk-deepseek"), &["authorization"])]
 #[case::deepseek_forwards_caller_authorization(
@@ -88,6 +91,7 @@ async fn credentials_become_exactly_one_auth_header(
 #[case::anthropic("anthropic")]
 #[case::azure_ai("azure_ai")]
 #[case::deepseek("deepseek")]
+#[case::minimax("minimax")]
 #[tokio::test]
 async fn a_call_without_credentials_fails_before_sending(
     call: MessagesCall,
@@ -123,6 +127,9 @@ async fn a_call_without_credentials_fails_before_sending(
     "/v1/messages",
     "/v1/messages"
 )]
+#[case::minimax(MODEL, Some("minimax"), "", "/v1/messages")]
+#[case::minimax_prefix("minimax/claude-sonnet-4-5", None, "/v1/messages", "/v1/messages")]
+#[case::minimax_endpoint_slash(MODEL, Some("minimax"), "/v1/messages/", "/v1/messages")]
 #[case::azure_ai(MODEL, Some("azure_ai"), "", "/anthropic/v1/messages")]
 #[case::deepseek(MODEL, Some("deepseek"), "", "/anthropic/v1/messages")]
 #[case::deepseek_openai_compatible_base(MODEL, Some("deepseek"), "/beta", "/anthropic/v1/messages")]
@@ -301,6 +308,7 @@ fn sent_betas(request: &wiremock::Request) -> BetaSet {
 #[tokio::test]
 async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
     call: MessagesCall,
+    #[values("anthropic", "minimax")] provider: &str,
     #[case] fields: Value,
     #[case] features: &[AnthropicBeta],
 ) {
@@ -312,6 +320,7 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
 
     run_message(with_fields(
         MessagesCall {
+            custom_llm_provider: Some(provider.into()),
             api_key: Some("sk".into()),
             api_base: Some(upstream.uri()),
             extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
@@ -362,6 +371,7 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
 #[case::anthropic("anthropic")]
 #[case::azure_ai("azure_ai")]
 #[case::deepseek("deepseek")]
+#[case::minimax("minimax")]
 #[tokio::test]
 async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
     let upstream = upstream([message_response()]).await;
@@ -452,6 +462,7 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i
 #[tokio::test]
 async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
     call: MessagesCall,
+    #[values("anthropic", "minimax")] provider: &str,
     #[case] thinking: Value,
     #[case] expected: Value,
 ) {
@@ -459,6 +470,7 @@ async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
 
     run_message(with_fields(
         MessagesCall {
+            custom_llm_provider: Some(provider.into()),
             api_key: Some("sk".into()),
             api_base: Some(upstream.uri()),
             shaping: MessagesShaping {
@@ -517,6 +529,7 @@ async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
 #[tokio::test]
 async fn reasoning_is_translated_by_the_model_capabilities(
     call: MessagesCall,
+    #[values("anthropic", "minimax")] provider: &str,
     #[case] capabilities: MessagesModelCapabilities,
     #[case] fields: Value,
     #[case] expected: Value,
@@ -525,6 +538,7 @@ async fn reasoning_is_translated_by_the_model_capabilities(
 
     run_message(with_fields(
         MessagesCall {
+            custom_llm_provider: Some(provider.into()),
             api_key: Some("sk".into()),
             api_base: Some(upstream.uri()),
             shaping: MessagesShaping {
@@ -592,6 +606,7 @@ async fn replayed_history_is_cleaned_before_sending(
 #[case::anthropic("anthropic")]
 #[case::azure("azure_ai")]
 #[case::deepseek("deepseek")]
+#[case::minimax("minimax")]
 #[tokio::test]
 async fn metadata_is_reduced_to_the_user_id(call: MessagesCall, #[case] provider: &str) {
     let upstream = upstream([message_response()]).await;
@@ -703,11 +718,16 @@ async fn vertex_ai_addresses_the_model_in_the_url_and_not_in_the_body(
 #[case::numeric_user_id(json!({"metadata": {"user_id": 7}}))]
 #[case::missing_max_tokens(json!({"max_tokens": null}))]
 #[tokio::test]
-async fn an_invalid_request_fails_before_sending(call: MessagesCall, #[case] fields: Value) {
+async fn an_invalid_request_fails_before_sending(
+    call: MessagesCall,
+    #[values("anthropic", "minimax")] provider: &str,
+    #[case] fields: Value,
+) {
     let upstream = upstream([message_response()]).await;
 
     let error = run(with_fields(
         MessagesCall {
+            custom_llm_provider: Some(provider.into()),
             api_key: Some("sk".into()),
             api_base: Some(upstream.uri()),
             ..call
@@ -721,6 +741,45 @@ async fn an_invalid_request_fails_before_sending(call: MessagesCall, #[case] fie
     assert!(received(&upstream).await.is_empty());
 }
 
+#[rstest]
+#[case::billing_string(json!("x-anthropic-billing-header: private"), None)]
+#[case::billing_blocks(json!([{ "type": "text", "text": "x-anthropic-billing-header: private" }]), None)]
+#[case::mixed_blocks(
+    json!([{ "type": "text", "text": "x-anthropic-billing-header: private" }, { "type": "text", "text": "keep" }]),
+    Some(json!([{ "type": "text", "text": "keep" }]))
+)]
+#[case::ordinary_system(json!("keep"), Some(json!("keep")))]
+#[tokio::test]
+async fn minimax_strips_only_billing_system_blocks(
+    call: MessagesCall,
+    #[case] system: Value,
+    #[case] expected: Option<Value>,
+) {
+    let upstream = upstream([message_response()]).await;
+    let fields = json!({
+        "system": system,
+        "tools": [{"name": "echo", "input_schema": {"type": "object"}}],
+        "messages": [{"role": "user", "content": [{
+            "type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}
+        }]}]
+    });
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some("minimax".into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            ..call
+        },
+        fields.clone(),
+    ))
+    .await;
+
+    let sent = only_request(&upstream).await.json();
+    assert_eq!(sent.get("system"), expected.as_ref());
+    assert_eq!(sent["tools"], fields["tools"]);
+    assert_eq!(sent["messages"], fields["messages"]);
+}
+
 #[rstest]
 #[case::azure("azure_ai", &[], json!([
     {"type": "text", "text": "top level"},

litellm-rust/crates/inference-messages/tests/messages/secrets.rs

@@ -25,6 +25,13 @@ use super::*;
     "/anthropic/v1/messages",
     &["DEEPSEEK_API_KEY", "DEEPSEEK_ANTHROPIC_API_BASE", "DEEPSEEK_API_BASE"]
 )]
+#[case::minimax(
+    "minimax",
+    "MINIMAX_API_KEY",
+    "MINIMAX_API_BASE",
+    "/v1/messages",
+    &["MINIMAX_API_KEY", "MINIMAX_API_BASE"]
+)]
 #[tokio::test]
 async fn the_credential_and_base_come_from_the_secret_source(
     call: MessagesCall,

litellm-rust/crates/inference-messages/tests/messages/stream.rs

@@ -150,9 +150,21 @@ async fn stream_through(host: &RecordingStreamHost) -> Result<MessagesOutput, Er
 
 #[rstest]
 #[tokio::test]
-async fn upstream_headers_are_on_the_stream_head_before_the_first_chunk(call: MessagesCall) {
+async fn upstream_headers_are_on_the_stream_head_before_the_first_chunk(
+    call: MessagesCall,
+    #[values("anthropic", "minimax")] provider: &str,
+) {
     let upstream = upstream([sse_response()]).await;
-    let host = RecordingStreamHost::new(streaming(call, upstream.uri()), usize::MAX);
+    let host = RecordingStreamHost::new(
+        streaming(
+            MessagesCall {
+                custom_llm_provider: Some(provider.into()),
+                ..call
+            },
+            upstream.uri(),
+        ),
+        usize::MAX,
+    );
 
     let outcome = stream_through(&host).await.expect("streamed call succeeds");
 

litellm-rust/crates/inference-messages/tests/messages/tencent.rs

@@ -0,0 +1,260 @@
+use std::ops::ControlFlow;
+
+use bytes::Bytes;
+use litellm_llms::base_llm::messages::context::MessagesModelCapabilities;
+use litellm_llms_types::providers::anthropic::DEFAULT_HEADERS;
+use rstest::{fixture, rstest};
+
+use super::*;
+
+#[fixture]
+fn tencent_call(call: MessagesCall) -> MessagesCall {
+    MessagesCall {
+        custom_llm_provider: Some("tencent".into()),
+        api_key: Some("deployment-key".into()),
+        ..call
+    }
+}
+
+#[rstest]
+#[case::deployment_key(&[], "x-api-key", "deployment-key")]
+#[case::forwarded_key(&[("X-Api-Key", "caller-key")], "x-api-key", "caller-key")]
+#[case::forwarded_bearer(&[("Authorization", "Bearer caller")], "authorization", "Bearer caller")]
+#[tokio::test]
+async fn credentials_reach_the_selected_provider(
+    tencent_call: MessagesCall,
+    #[case] forwarded: &[(&str, &str)],
+    #[case] header: &str,
+    #[case] credential: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+    run_message(MessagesCall {
+        api_base: Some(upstream.uri()),
+        extra_headers: headers(forwarded.iter().copied()),
+        ..tencent_call
+    })
+    .await;
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values(header), [credential]);
+    let version = DEFAULT_HEADERS
+        .iter()
+        .find_map(|(name, value)| (*name == "anthropic-version").then_some(*value))
+        .unwrap();
+    assert_eq!(sent.header_values("anthropic-version"), [version]);
+    assert_eq!(sent.json()["model"], MODEL);
+}
+
+#[rstest]
+#[tokio::test]
+async fn credentials_and_base_resolve_from_the_secret_source(tencent_call: MessagesCall) {
+    let upstream = upstream([message_response()]).await;
+    let base = format!("{}/v1/chat/completions/", upstream.uri());
+    let secrets = Arc::new(RecordingSecrets::new([
+        ("TENCENT_API_KEY", "secret-key"),
+        ("TENCENT_ANTHROPIC_API_BASE", base.as_str()),
+        ("TENCENT_API_BASE", UNREACHABLE_BASE),
+    ]));
+    run_with(
+        secrets,
+        MessagesCall {
+            api_key: None,
+            ..tencent_call
+        },
+    )
+    .await
+    .unwrap();
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v1/messages");
+    assert_eq!(sent.header("x-api-key"), Some("secret-key"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn missing_credentials_fail_before_sending(tencent_call: MessagesCall) {
+    let upstream = upstream([message_response()]).await;
+    let error = run(MessagesCall {
+        api_key: None,
+        api_base: Some(upstream.uri()),
+        ..tencent_call
+    })
+    .await
+    .unwrap_err();
+    assert!(matches!(
+        error,
+        Error::Auth(litellm_auth::Error::MissingApiKey {
+            environment_variable: "TENCENT_API_KEY",
+            ..
+        })
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::missing_max_tokens(json!({"max_tokens": null}))]
+#[case::invalid_metadata(json!({"metadata": {"user_id": 7}}))]
+#[case::invalid_reasoning_effort(json!({"reasoning_effort": "unknown"}))]
+#[tokio::test]
+async fn malformed_requests_fail_before_sending(tencent_call: MessagesCall, #[case] fields: Value) {
+    let upstream = upstream([message_response()]).await;
+    let error = run(with_fields(
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            ..tencent_call
+        },
+        fields,
+    ))
+    .await
+    .unwrap_err();
+    assert!(error.is_request(), "{error:?}");
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::legacy(false, json!({"type": "enabled", "budget_tokens": 4095}))]
+#[case::adaptive(true, json!({"type": "adaptive", "display": "summarized"}))]
+#[tokio::test]
+async fn reasoning_effort_becomes_model_appropriate_thinking(
+    tencent_call: MessagesCall,
+    #[case] adaptive: bool,
+    #[case] thinking: Value,
+) {
+    let upstream = upstream([message_response()]).await;
+    run_message(with_fields(
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            shaping: MessagesShaping {
+                capabilities: MessagesModelCapabilities {
+                    supports_reasoning: true,
+                    supports_adaptive_thinking: adaptive,
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            ..tencent_call
+        },
+        json!({"max_tokens": 4096, "reasoning_effort": "high"}),
+    ))
+    .await;
+    let sent = only_request(&upstream).await.json();
+    assert_eq!(sent["thinking"], thinking);
+    assert_eq!(sent.get("reasoning_effort"), None);
+    assert_eq!(
+        sent.get("output_config"),
+        adaptive.then(|| json!({"effort": "high"})).as_ref()
+    );
+}
+
+#[rstest]
+#[case::encrypted_reasoning(json!({"type": "thinking", "thinking": "private", "signature": "litellm_encrypted_reasoning:private"}))]
+#[case::orphaned_advisor(json!({"type": "server_tool_use", "name": "advisor", "id": "srvtoolu_advisor", "input": {}}))]
+#[tokio::test]
+async fn replayed_history_is_cleaned_without_losing_text(
+    tencent_call: MessagesCall,
+    #[case] block: Value,
+) {
+    let upstream = upstream([message_response()]).await;
+    run_message(with_fields(
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            ..tencent_call
+        },
+        json!({"messages": [
+            {"role": "assistant", "content": [block, {"type": "text", "text": "keep"}]},
+            {"role": "user", "content": "continue"}
+        ]}),
+    ))
+    .await;
+    assert_eq!(
+        only_request(&upstream).await.json()["messages"],
+        json!([
+            {"role": "assistant", "content": [{"type": "text", "text": "keep"}]},
+            {"role": "user", "content": "continue"}
+        ])
+    );
+}
+
+#[rstest]
+#[case::billing_string(json!("x-anthropic-billing-header: private"), None)]
+#[case::mixed_blocks(json!([
+    {"type": "text", "text": "x-anthropic-billing-header: private"},
+    {"type": "text", "text": "keep"}
+]), Some(json!([{ "type": "text", "text": "keep" }])))]
+#[case::ordinary_system(json!("keep"), Some(json!("keep")))]
+#[tokio::test]
+async fn billing_is_removed_and_tools_and_cache_hints_are_preserved(
+    tencent_call: MessagesCall,
+    #[case] system: Value,
+    #[case] expected: Option<Value>,
+) {
+    let upstream = upstream([message_response()]).await;
+    let fields = json!({
+        "system": system,
+        "tools": [{"name": "echo", "input_schema": {"type": "object"}}],
+        "messages": [{"role": "user", "content": [{
+            "type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}
+        }]}]
+    });
+    run_message(with_fields(
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            ..tencent_call
+        },
+        fields.clone(),
+    ))
+    .await;
+    let sent = only_request(&upstream).await.json();
+    assert_eq!(sent.get("system"), expected.as_ref());
+    assert_eq!(sent["tools"], fields["tools"]);
+    assert_eq!(sent["messages"], fields["messages"]);
+}
+
+#[derive(Default)]
+struct StreamSink(Mutex<Vec<u8>>);
+
+impl litellm_host_native::in_process::StreamConsumer<Messages> for StreamSink {
+    async fn open_stream(
+        &self,
+        _: litellm_inference_messages::route::MessagesStreamHead,
+    ) -> Result<ControlFlow<()>, Error> {
+        Ok(ControlFlow::Continue(()))
+    }
+
+    async fn send_chunk(&self, chunk: Bytes) -> Result<ControlFlow<()>, Error> {
+        self.0.lock().unwrap().extend_from_slice(&chunk);
+        Ok(ControlFlow::Continue(()))
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn streaming_relays_provider_events_unchanged(tencent_call: MessagesCall) {
+    let events = concat!(
+        "event: message_start\ndata: {\"type\":\"message_start\"}\n\n",
+        "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"OK\"}}\n\n",
+        "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
+    );
+    let upstream =
+        upstream([ResponseTemplate::new(200).set_body_raw(events, "text/event-stream")]).await;
+    let host = LocalMessagesHost::new(with_fields(
+        MessagesCall {
+            api_base: Some(upstream.uri()),
+            ..tencent_call
+        },
+        json!({"stream": true}),
+    ));
+    let sink = StreamSink::default();
+    let outcome = litellm_host_native::in_process::run_hosted(
+        machine(Arc::new(RecordingSecrets::empty()))(host.request().unwrap()),
+        litellm_host_native::in_process::Host {
+            services: &(),
+            interceptors: &host,
+            stream: &sink,
+            observers: None,
+        },
+    )
+    .await
+    .unwrap();
+    assert!(matches!(outcome, MessagesOutput::StreamEnded));
+    assert_eq!(sink.0.into_inner().unwrap(), events.as_bytes());
+    assert_eq!(only_request(&upstream).await.json()["stream"], true);
+}

litellm-rust/crates/inference-messages/tests/openrouter/main.rs

@@ -0,0 +1,147 @@
+use std::{sync::Arc, time::Duration};
+
+use futures_util::TryStreamExt;
+use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping};
+use litellm_inference_testing::{RecordingSecrets, no_secrets};
+use rstest::{fixture, rstest};
+use serde_json::json;
+use wiremock::ResponseTemplate;
+
+#[path = "../support/mod.rs"]
+mod support;
+use support::*;
+
+#[fixture]
+fn call() -> MessagesCall {
+    MessagesCall {
+        body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})).unwrap(),
+        api_key: None,
+        api_base: None,
+        custom_llm_provider: None,
+        litellm_params: Default::default(),
+        extra_headers: None,
+        provider_specific_header: None,
+        timeout: Some(Duration::from_secs(5)),
+        shaping: MessagesShaping::default(),
+    }
+}
+
+fn message_response() -> ResponseTemplate {
+    json_response(
+        json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}),
+    )
+}
+
+#[rstest]
+#[case::explicit(Some("explicit"), None, "Bearer explicit")]
+#[case::managed(None, None, "Bearer managed")]
+#[case::forwarded(None, Some("bEaReR caller"), "bEaReR caller")]
+#[case::blank(Some("explicit"), Some("Bearer "), "Bearer explicit")]
+#[case::malformed(Some("explicit"), Some("invali\u{00e9}auth"), "Bearer explicit")]
+#[tokio::test]
+async fn only_a_valid_bearer_can_override_the_configured_key(
+    call: MessagesCall,
+    #[case] api_key: Option<&str>,
+    #[case] authorization: Option<&str>,
+    #[case] expected: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+    let extra_headers = [("x-api-key".to_string(), json!("caller-anthropic-key"))]
+        .into_iter()
+        .chain(authorization.map(|value| ("Authorization".to_string(), json!(value))))
+        .collect();
+    let secrets = Arc::new(RecordingSecrets::new([("OPENROUTER_API_KEY", "managed")]));
+    let output = messages_route(secrets)
+        .execute(
+            MessagesCall {
+                api_key: api_key.map(str::to_string),
+                api_base: Some(format!("{}/api/v1", upstream.uri())),
+                extra_headers: Some(extra_headers),
+                ..call
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+    assert!(matches!(output, MessagesCallResponse::Complete(_)));
+    let request = only_request(&upstream).await;
+    assert_eq!(request.url.path(), "/api/v1/messages");
+    assert_eq!(request.header_values("authorization"), [expected]);
+    assert_eq!(request.header("x-api-key"), None);
+    assert_eq!(request.json()["model"], "anthropic/test-model");
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_anthropic_key_alone_is_not_an_openrouter_credential(call: MessagesCall) {
+    let upstream = upstream([message_response()]).await;
+    let result = messages_route(no_secrets())
+        .execute(
+            MessagesCall {
+                api_base: Some(upstream.uri()),
+                extra_headers: Some(
+                    json!({"x-api-key": "caller-key"})
+                        .as_object()
+                        .unwrap()
+                        .clone(),
+                ),
+                ..call
+            },
+            &(),
+            None,
+        )
+        .await;
+    assert!(matches!(
+        result,
+        Err(Error::Auth(litellm_auth::Error::MissingApiKey {
+            provider: "OpenRouter",
+            ..
+        }))
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_secret_manager_supplies_the_native_base_and_key(call: MessagesCall) {
+    let upstream = upstream([message_response()]).await;
+    let base = format!("{}/api/v1", upstream.uri());
+    let secrets = Arc::new(RecordingSecrets::new([
+        ("OPENROUTER_API_KEY", "managed"),
+        ("OPENROUTER_API_BASE", base.as_str()),
+    ]));
+    messages_route(secrets)
+        .execute(call, &(), None)
+        .await
+        .unwrap();
+    let request = only_request(&upstream).await;
+    assert_eq!(request.url.path(), "/api/v1/messages");
+    assert_eq!(request.header("authorization"), Some("Bearer managed"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn streaming_relay_preserves_every_event_and_billed_usage(call: MessagesCall) {
+    let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":\"0.001\",\"cost_details\":\"unexpected\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
+    let upstream =
+        upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
+    let streamed = MessagesCall {
+        body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}], "stream": true})).unwrap(),
+        api_key: Some("key".into()),
+        api_base: Some(upstream.uri()),
+        ..call
+    };
+    let output = messages_route(no_secrets())
+        .execute(streamed, &(), None)
+        .await
+        .unwrap();
+    let MessagesCallResponse::Stream { chunks, .. } = output else {
+        panic!("expected SSE");
+    };
+    assert_eq!(
+        chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
+        sse.as_bytes()
+    );
+    assert_eq!(only_request(&upstream).await.json()["stream"], true);
+}

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- Reuse Anthropic Messages shaping, reasoning translation, and beta headers
+- Require a workspace header and use an API key or AWS SigV4
+- Keep connection fields out of the upstream request body
+- Drop `context_management` by default, with per-request `claude_platform_unsupported_params` overrides
+
+# references
+
+- Python counterpart `litellm/llms/bedrock/claude_platform/messages_transformation.py`
+- https://platform.claude.com/docs/en/build-with-claude/claude-platform-on-aws

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -0,0 +1,395 @@
+use std::sync::LazyLock;
+
+use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue};
+use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_http::request::{header_value, with_header, without_headers};
+use litellm_llms_types::{
+    formats::messages::MessagesRequest,
+    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+};
+use litellm_router_types::LitellmParams;
+use serde_json::Value;
+
+use crate::{
+    Error, ErrorDetail,
+    anthropic::messages::{
+        handler::shape_anthropic_messages_request,
+        transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+    },
+    base_llm::{
+        auth::{AuthScheme, Headers, ValidatedEnvironment},
+        messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
+    },
+    bedrock::common_utils::validated_region,
+};
+
+const SERVICE: &str = "aws-external-anthropic";
+const BASE_ENVS: &[&str] = &["ANTHROPIC_AWS_BASE_URL", "ANTHROPIC_AWS_API_BASE"];
+const WORKSPACE_ENVS: &[&str] = &["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"];
+const WORKSPACE_FIELDS: &[&str] = &[
+    "workspace_id",
+    "aws_workspace_id",
+    "anthropic-workspace-id",
+    "anthropic_workspace_id",
+];
+const WORKSPACE_HEADER: &str = "anthropic-workspace-id";
+static WORKSPACE: ParamSpec = ParamSpec {
+    setting: "workspace_id",
+    wire: WORKSPACE_FIELDS,
+    module_global: None,
+    env: WORKSPACE_ENVS,
+};
+const UNSUPPORTED_OVERRIDE: &str = "claude_platform_unsupported_params";
+
+pub struct BedrockClaudePlatformMessagesConfig;
+pub const BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG: BedrockClaudePlatformMessagesConfig =
+    BedrockClaudePlatformMessagesConfig;
+
+impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
+    fn shape_request(
+        &self,
+        request: MessagesRequest,
+        reasoning_auto_summary: bool,
+    ) -> Result<MessagesRequest, Error> {
+        shape_anthropic_messages_request(request, reasoning_auto_summary)
+    }
+
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        let workspace = header_value(&headers, WORKSPACE_HEADER)
+            .filter(|value| !value.trim().is_empty())
+            .or_else(|| {
+                WORKSPACE_FIELDS.iter().find_map(|field| {
+                    params
+                        .extra
+                        .get(*field)?
+                        .as_str()
+                        .filter(|value| !value.trim().is_empty())
+                })
+            });
+        let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
+            .ok_or_else(|| Error::Auth(WORKSPACE.missing("Claude Platform on AWS")))?;
+        let headers = with_header(headers, WORKSPACE_HEADER, workspace);
+        let key = resolve_non_empty(api_key, env, &["ANTHROPIC_AWS_API_KEY"]);
+        let headers = without_headers(headers, &["authorization"]);
+        if header_value(&headers, "x-api-key").is_some_and(|value| !value.trim().is_empty()) {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        if let Some(key) = key {
+            return Ok(ValidatedEnvironment {
+                headers: without_headers(headers, &["x-api-key"]),
+                auth: AuthScheme::Credential {
+                    placement: CredentialPlacement::Header("x-api-key"),
+                    secret: SecretValue::new(key),
+                },
+            });
+        }
+        let base = resolve_non_empty(params.api_base.as_deref(), env, BASE_ENVS);
+        Ok(ValidatedEnvironment {
+            headers: without_headers(headers, &["x-api-key"]),
+            auth: AuthScheme::AwsSigV4 {
+                region: platform_region(base.as_deref(), params, env)?,
+                service: SERVICE,
+                credentials: Box::new(AwsCredentialSource::from_params(&params.aws, env)),
+            },
+        })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let base = resolve_non_empty(
+            api_base
+                .filter(|base| !base.trim().is_empty())
+                .or(params.api_base.as_deref()),
+            env,
+            BASE_ENVS,
+        );
+        let base = match base {
+            Some(base) => base,
+            None => format!(
+                "https://{SERVICE}.{}.api.aws",
+                platform_region(None, params, env)?
+            ),
+        };
+        let base = base.trim_end_matches('/');
+        Ok(if base.ends_with(MESSAGES_PATH) {
+            base.into()
+        } else {
+            format!("{base}{MESSAGES_PATH}")
+        })
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        let body = serde_json::to_value(request).map_err(invalid_request)?;
+        let filtered = serde_json::from_value(filter_body(body, false)).map_err(invalid_request)?;
+        transform_messages_request(filtered, context)
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        static NAMES: LazyLock<Box<[&'static str]>> = LazyLock::new(|| {
+            ["ANTHROPIC_AWS_API_KEY"]
+                .into_iter()
+                .chain(BASE_ENVS.iter().copied())
+                .chain(WORKSPACE_ENVS.iter().copied())
+                .chain(["AWS_DEFAULT_REGION"])
+                .chain(AwsParams::secret_names().iter().copied())
+                .collect()
+        });
+        &NAMES
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        let headers = match WORKSPACE_FIELDS.iter().find_map(|field| {
+            request
+                .params
+                .extra
+                .get(*field)?
+                .as_str()
+                .filter(|value| !value.trim().is_empty())
+        }) {
+            Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
+            None => headers,
+        };
+        update_headers_with_anthropic_beta(headers, request)
+    }
+
+    fn wire_body(&self, body: Value) -> Value {
+        filter_body(body, true)
+    }
+}
+
+fn platform_region(
+    base: Option<&str>,
+    params: &LitellmParams,
+    env: &dyn Fn(&str) -> Option<String>,
+) -> Result<String, Error> {
+    let host_region = base.and_then(|base| {
+        let url = url::Url::parse(base).ok()?;
+        url.host_str()?
+            .strip_prefix("aws-external-anthropic.")?
+            .strip_suffix(".api.aws")
+            .map(str::to_string)
+    });
+    let region = host_region
+        .or_else(|| resolve_aws_region(None, &params.aws, env))
+        .or_else(|| resolve_non_empty(None, env, &["AWS_DEFAULT_REGION"]))
+        .ok_or_else(|| Error::Auth(AwsParams::REGION.missing("Claude Platform on AWS")))?;
+    validated_region(region)
+}
+
+fn filter_body(body: Value, strip_connection_fields: bool) -> Value {
+    let Value::Object(fields) = body else {
+        return body;
+    };
+    let unsupported: Vec<&str> = fields
+        .get(UNSUPPORTED_OVERRIDE)
+        .and_then(Value::as_array)
+        .map(|values| values.iter().filter_map(Value::as_str).collect())
+        .unwrap_or_else(|| vec!["context_management"]);
+    let retained = fields
+        .iter()
+        .filter(|(name, _)| {
+            !unsupported.contains(&name.as_str())
+                && !(strip_connection_fields
+                    && (WORKSPACE_FIELDS.contains(&name.as_str())
+                        || name.starts_with("aws_")
+                        || name.as_str() == UNSUPPORTED_OVERRIDE))
+        })
+        .map(|(name, value)| {
+            let value = if name == "model" {
+                value
+                    .as_str()
+                    .map(|model| {
+                        Value::String(
+                            model
+                                .strip_prefix("claude_platform/")
+                                .unwrap_or(model)
+                                .into(),
+                        )
+                    })
+                    .unwrap_or_else(|| value.clone())
+            } else {
+                value.clone()
+            };
+            (name.clone(), value)
+        })
+        .collect();
+    Value::Object(retained)
+}
+
+fn invalid_request(error: serde_json::Error) -> Error {
+    Error::InvalidRequest(ErrorDetail::invalid("Messages request", error))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use litellm_llms_types::providers::anthropic::AnthropicBeta;
+    use rstest::rstest;
+    use serde_json::json;
+
+    #[rstest]
+    fn workspace_is_required_before_any_upstream_request() {
+        assert_eq!(
+            BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+                .validate_environment(
+                    vec![],
+                    Some("key"),
+                    "claude-test",
+                    &LitellmParams::default(),
+                    &|_| None
+                )
+                .unwrap_err(),
+            Error::Auth(WORKSPACE.missing("Claude Platform on AWS"))
+        );
+    }
+
+    #[rstest]
+    #[case::valid("forwarded", true)]
+    #[case::blank(" ", false)]
+    fn forwarded_keys_win_and_blank_keys_fall_back(#[case] key: &str, #[case] forwarded: bool) {
+        let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+            .validate_environment(
+                vec![
+                    ("X-Api-Key".into(), key.into()),
+                    ("authorization".into(), "stale".into()),
+                ],
+                Some("explicit"),
+                "claude-test",
+                &LitellmParams::default(),
+                &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
+            )
+            .unwrap();
+        assert_eq!(
+            header_value(&environment.headers, WORKSPACE_HEADER),
+            Some("workspace")
+        );
+        assert_eq!(header_value(&environment.headers, "authorization"), None);
+        if forwarded {
+            assert_eq!(header_value(&environment.headers, "x-api-key"), Some(key));
+            assert!(matches!(environment.auth, AuthScheme::Forwarded));
+        } else {
+            assert_eq!(header_value(&environment.headers, "x-api-key"), None);
+            assert!(
+                matches!(environment.auth, AuthScheme::Credential { secret, placement: CredentialPlacement::Header("x-api-key") } if secret.expose() == "explicit")
+            );
+        }
+    }
+
+    #[rstest]
+    #[case::region("us-west-2", true)]
+    #[case::injection("us-east-1.attacker.test/steal?", false)]
+    fn generated_endpoint_regions_are_validated(#[case] region: &str, #[case] valid: bool) {
+        let result = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.get_complete_url(
+            None,
+            "claude-test",
+            &LitellmParams::default(),
+            false,
+            &|name| (name == "AWS_DEFAULT_REGION").then(|| region.into()),
+        );
+        if valid {
+            assert_eq!(
+                result.unwrap(),
+                format!("https://{SERVICE}.{region}.api.aws/v1/messages")
+            );
+        } else {
+            assert!(matches!(result, Err(Error::InvalidRequest(_))));
+        }
+    }
+
+    #[rstest]
+    #[case::explicit_regional_endpoint(
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "us-east-1",
+        "us-west-2"
+    )]
+    fn public_regional_host_precedes_configured_region(
+        #[case] api_base: &str,
+        #[case] configured_region: &str,
+        #[case] expected_region: &str,
+    ) {
+        let params = LitellmParams {
+            api_base: Some(api_base.into()),
+            aws: AwsParams {
+                aws_region_name: Some(configured_region.into()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let env = |name: &str| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into());
+        let url = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+            .get_complete_url(None, "claude-test", &params, false, &env)
+            .unwrap();
+        let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+            .validate_environment(vec![], None, "claude-test", &params, &env)
+            .unwrap();
+        assert_eq!(
+            url,
+            "https://aws-external-anthropic.us-west-2.api.aws/v1/messages"
+        );
+        assert!(
+            matches!(environment.auth, AuthScheme::AwsSigV4 { region, service: SERVICE, .. } if region == expected_region)
+        );
+    }
+
+    #[rstest]
+    #[case::default(json!({}), false)]
+    #[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]
+    fn unsupported_fields_are_filtered_before_beta_selection(
+        #[case] extra: Value,
+        #[case] keep_context: bool,
+    ) {
+        let fields = json!({"model": "claude_platform/claude-test", "max_tokens": 16, "messages": [], "workspace_id": "workspace", "aws_region_name": "us-west-2", "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}});
+        let input = Value::Object(
+            fields
+                .as_object()
+                .unwrap()
+                .clone()
+                .into_iter()
+                .chain(extra.as_object().unwrap().clone())
+                .collect(),
+        );
+        let request = serde_json::from_value(input).unwrap();
+        let transformed = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+            .transform_anthropic_messages_request(request, &MessagesTransformContext::default())
+            .unwrap();
+        let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &transformed);
+        let expected_beta = AnthropicBeta::ContextManagement20250627.to_string();
+        assert_eq!(
+            header_value(&headers, "anthropic-beta")
+                .is_some_and(|value| value.split(',').any(|beta| beta == expected_beta)),
+            keep_context
+        );
+        let wire = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
+            .wire_body(serde_json::to_value(transformed).unwrap());
+        assert_eq!(wire["model"], "claude-test");
+        assert_eq!(wire.get("context_management").is_some(), keep_context);
+        assert_eq!(wire.get("workspace_id"), None);
+        assert_eq!(wire.get("aws_region_name"), None);
+        assert_eq!(wire.get(UNSUPPORTED_OVERRIDE), None);
+    }
+}

litellm-rust/crates/llms/src/bedrock/claude_platform/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/llms/src/bedrock/common_utils.rs

@@ -0,0 +1,16 @@
+use crate::{Error, ErrorDetail};
+
+pub(crate) fn validated_region(region: String) -> Result<String, Error> {
+    if !region.is_empty()
+        && region
+            .bytes()
+            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
+    {
+        return Ok(region);
+    }
+    Err(Error::InvalidRequest(ErrorDetail::InvalidValue {
+        field: "aws_region_name",
+        expected: "lowercase letters, digits, and hyphens",
+        actual: region.into(),
+    }))
+}

litellm-rust/crates/llms/src/bedrock/mod.rs

@@ -1,3 +1,5 @@
 pub mod audio_transcription;
 pub mod chat;
+pub mod claude_platform;
+pub(crate) mod common_utils;
 pub mod messages;

litellm-rust/crates/llms/src/bedrock_mantle/messages/AGENTS.md

@@ -0,0 +1,10 @@
+# rules
+
+- Reuse Anthropic Messages shaping and reasoning translation
+- Use bearer credentials or AWS SigV4 for the Mantle Messages endpoint
+- Filter and rename caller betas using the Mantle beta policy
+
+# references
+
+- Python counterpart `litellm/llms/bedrock/messages/mantle_transformation.py`
+- https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html

litellm-rust/crates/llms/src/bedrock_mantle/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs

@@ -0,0 +1,528 @@
+use std::sync::LazyLock;
+
+use litellm_auth::{AwsParams, CredentialPlacement, SecretValue};
+use litellm_auth_aws::AwsCredentialSource;
+use litellm_auth_aws::bedrock_model_id_and_region;
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_http::request::{with_header, without_headers};
+use litellm_llms_types::{
+    formats::messages::MessagesRequest,
+    providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
+};
+use litellm_router_types::LitellmParams;
+use serde_json::Value;
+
+use crate::{
+    Error,
+    anthropic::{
+        common_utils::{existing_betas, merge_beta_headers},
+        messages::{
+            handler::shape_anthropic_messages_request,
+            transformation::{provider_feature_betas, transform_messages_request},
+        },
+    },
+    base_llm::{
+        auth::AuthScheme,
+        messages::{
+            context::MessagesTransformContext,
+            transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment},
+        },
+    },
+    bedrock::common_utils::validated_region,
+};
+
+const API_BASE_ENV: &str = "BEDROCK_MANTLE_API_BASE";
+const REGION_ENVS: &[&str] = &[
+    "BEDROCK_MANTLE_REGION",
+    "AWS_REGION_NAME",
+    "AWS_REGION",
+    "AWS_DEFAULT_REGION",
+];
+const KEY_ENVS: &[&str] = &["BEDROCK_MANTLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK"];
+const MESSAGES_PATH: &str = "/anthropic/v1/messages";
+const BASE_SUFFIXES: &[&str] = &[
+    MESSAGES_PATH,
+    "/v1/messages",
+    "/messages",
+    "/anthropic/v1",
+    "/openai/v1",
+    "/v1",
+];
+
+pub struct BedrockMantleAnthropicMessagesConfig;
+pub const BEDROCK_MANTLE_MESSAGES_CONFIG: BedrockMantleAnthropicMessagesConfig =
+    BedrockMantleAnthropicMessagesConfig;
+
+impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        model: &str,
+        params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        let headers: Headers = headers
+            .into_iter()
+            .filter(|(name, _)| {
+                !name.eq_ignore_ascii_case("authorization")
+                    && !name.eq_ignore_ascii_case("x-api-key")
+            })
+            .collect();
+        let headers = match params
+            .extra
+            .get("aws_bedrock_project_id")
+            .and_then(Value::as_str)
+        {
+            Some(project) => with_header(headers, "anthropic-workspace-id", project.to_string()),
+            None => headers,
+        };
+        let auth = match resolve_non_empty(api_key, env, KEY_ENVS) {
+            Some(token) => AuthScheme::Credential {
+                placement: CredentialPlacement::Bearer,
+                secret: SecretValue::new(token),
+            },
+            None => AuthScheme::AwsSigV4 {
+                region: mantle_region(params.api_base.as_deref(), model, params, env)?,
+                service: "bedrock",
+                credentials: Box::new(AwsCredentialSource::from_params(&params.aws, env)),
+            },
+        };
+        Ok(ValidatedEnvironment { headers, auth })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        model: &str,
+        params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let region = mantle_region(api_base, model, params, env)?;
+        let configured = mantle_base(api_base, params, env)
+            .unwrap_or_else(|| format!("https://bedrock-mantle.{region}.api.aws"));
+        let trimmed = configured.trim_end_matches('/');
+        let stripped = BASE_SUFFIXES
+            .iter()
+            .find_map(|suffix| trimmed.strip_suffix(suffix))
+            .unwrap_or(trimmed);
+        let base = if mantle_host_region(stripped).is_some() {
+            format!("https://bedrock-mantle.{region}.api.aws")
+        } else {
+            stripped.to_string()
+        };
+        Ok(format!("{base}{MESSAGES_PATH}"))
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        transform_messages_request(request, context)
+    }
+
+    fn shape_request(
+        &self,
+        request: MessagesRequest,
+        reasoning_auto_summary: bool,
+    ) -> Result<MessagesRequest, Error> {
+        shape_anthropic_messages_request(request, reasoning_auto_summary)
+    }
+
+    fn wire_body(&self, body: Value) -> Value {
+        match body {
+            Value::Object(fields) => Value::Object(
+                fields
+                    .into_iter()
+                    .filter_map(|(name, value)| match name.as_str() {
+                        "anthropic_version" | "anthropic_beta" => None,
+                        "model" => Some((
+                            name,
+                            value
+                                .as_str()
+                                .map(|model| Value::String(mantle_model(model).0))
+                                .unwrap_or(value),
+                        )),
+                        _ => Some((name, value)),
+                    })
+                    .collect(),
+            ),
+            other => other,
+        }
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        static NAMES: LazyLock<Box<[&'static str]>> = LazyLock::new(|| {
+            [
+                "BEDROCK_MANTLE_API_KEY",
+                "AWS_BEARER_TOKEN_BEDROCK",
+                API_BASE_ENV,
+            ]
+            .into_iter()
+            .chain(REGION_ENVS.iter().copied())
+            .chain(
+                AwsParams::secret_names()
+                    .iter()
+                    .copied()
+                    .filter(|name| !REGION_ENVS.contains(name)),
+            )
+            .collect()
+        });
+        &NAMES
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        let body_betas: BetaSet = request
+            .params
+            .extra
+            .get("anthropic_beta")
+            .and_then(Value::as_array)
+            .into_iter()
+            .flatten()
+            .filter_map(Value::as_str)
+            .filter_map(|beta| beta.parse::<AnthropicBeta>().ok())
+            .collect();
+        let caller: BetaSet = existing_betas(&headers)
+            .union(body_betas)
+            .into_iter()
+            .filter_map(|beta| match beta {
+                AnthropicBeta::Other(_) => Some(beta),
+                _ => beta.on(BetaProvider::BedrockMantle),
+            })
+            .collect();
+        let safeguards: BetaSet = request
+            .params
+            .extra
+            .contains_key("safeguards")
+            .then_some(AnthropicBeta::DangerousToolUse20260903)
+            .into_iter()
+            .collect();
+        merge_beta_headers(
+            without_headers(headers, &["anthropic-beta"]),
+            caller
+                .union(provider_feature_betas(request, BetaProvider::BedrockMantle))
+                .union(safeguards),
+        )
+    }
+}
+
+fn mantle_model(model: &str) -> (String, Option<String>) {
+    bedrock_model_id_and_region(model.strip_prefix("mantle/").unwrap_or(model))
+}
+
+fn mantle_host_region(base: &str) -> Option<String> {
+    let url = url::Url::parse(base).ok()?;
+    if !matches!(url.scheme(), "https" | "http") {
+        return None;
+    }
+    url.host_str()?
+        .strip_prefix("bedrock-mantle.")?
+        .strip_suffix(".api.aws")
+        .filter(|region| !region.contains('.') && !region.is_empty())
+        .map(str::to_string)
+}
+
+fn mantle_base(
+    base: Option<&str>,
+    params: &LitellmParams,
+    env: &dyn Fn(&str) -> Option<String>,
+) -> Option<String> {
+    let explicit = [
+        base,
+        params.api_base.as_deref(),
+        params.aws.aws_bedrock_runtime_endpoint.as_deref(),
+    ]
+    .into_iter()
+    .flatten()
+    .find(|base| !base.trim().is_empty());
+    resolve_non_empty(explicit, env, &[API_BASE_ENV])
+}
+
+fn mantle_region(
+    base: Option<&str>,
+    model: &str,
+    params: &LitellmParams,
+    env: &dyn Fn(&str) -> Option<String>,
+) -> Result<String, Error> {
+    let region = mantle_base(base, params, env)
+        .and_then(|base| mantle_host_region(&base))
+        .or_else(|| resolve_non_empty(params.aws.aws_region_name.as_deref(), &|_| None, &[]))
+        .or_else(|| mantle_model(model).1)
+        .or_else(|| resolve_non_empty(None, env, REGION_ENVS))
+        .unwrap_or_else(|| "us-east-1".into());
+    validated_region(region)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use litellm_http::request::header_value;
+    use rstest::rstest;
+    use serde_json::json;
+
+    #[rstest]
+    #[case::root("https://private.test", "https://private.test/anthropic/v1/messages")]
+    #[case::openai(
+        "https://private.test/openai/v1/",
+        "https://private.test/anthropic/v1/messages"
+    )]
+    #[case::complete(
+        "https://private.test/anthropic/v1/messages/",
+        "https://private.test/anthropic/v1/messages"
+    )]
+    #[case::tenant(
+        "https://private.test/tenant/v1",
+        "https://private.test/tenant/anthropic/v1/messages"
+    )]
+    fn endpoints_are_normalized(#[case] base: &str, #[case] expected: &str) {
+        assert_eq!(
+            BEDROCK_MANTLE_MESSAGES_CONFIG
+                .get_complete_url(
+                    Some(base),
+                    "claude-test",
+                    &LitellmParams::default(),
+                    true,
+                    &|_| None
+                )
+                .unwrap(),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::parameter("us-west-2", "claude-test")]
+    #[case::model("us-west-2", "us-west-2/claude-test")]
+    fn private_endpoints_keep_their_host_and_signing_region(
+        #[case] region: &str,
+        #[case] model: &str,
+    ) {
+        let params = LitellmParams {
+            aws: AwsParams {
+                aws_region_name: (model == "claude-test").then(|| region.into()),
+                aws_bedrock_runtime_endpoint: Some("https://private.test/anthropic/v1".into()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert_eq!(
+            BEDROCK_MANTLE_MESSAGES_CONFIG
+                .get_complete_url(None, model, &params, false, &|_| None)
+                .unwrap(),
+            "https://private.test/anthropic/v1/messages"
+        );
+        let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
+            .validate_environment(vec![], None, model, &params, &|_| None)
+            .unwrap();
+        assert!(
+            matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
+        );
+    }
+
+    #[rstest]
+    #[case::explicit_regional_endpoint(
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "us-east-1",
+        "us-west-2"
+    )]
+    fn public_regional_host_precedes_configured_region(
+        #[case] api_base: &str,
+        #[case] configured_region: &str,
+        #[case] expected_region: &str,
+    ) {
+        let params = LitellmParams {
+            api_base: Some(api_base.into()),
+            aws: AwsParams {
+                aws_region_name: Some(configured_region.into()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let url = BEDROCK_MANTLE_MESSAGES_CONFIG
+            .get_complete_url(None, "claude-test", &params, false, &|_| None)
+            .unwrap();
+        let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
+            .validate_environment(vec![], None, "claude-test", &params, &|_| None)
+            .unwrap();
+        assert_eq!(
+            url,
+            "https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages"
+        );
+        assert!(
+            matches!(environment.auth, AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == expected_region)
+        );
+    }
+
+    #[rstest]
+    #[case::aws_default_region("AWS_DEFAULT_REGION", "eu-west-1")]
+    fn aws_default_region_selects_mantle_endpoint(#[case] env_name: &str, #[case] region: &str) {
+        let url = BEDROCK_MANTLE_MESSAGES_CONFIG
+            .get_complete_url(
+                None,
+                "claude-test",
+                &LitellmParams::default(),
+                false,
+                &|name| (name == env_name).then(|| region.into()),
+            )
+            .unwrap();
+        assert_eq!(
+            url,
+            "https://bedrock-mantle.eu-west-1.api.aws/anthropic/v1/messages"
+        );
+    }
+
+    #[rstest]
+    #[case::aws_default_region("AWS_DEFAULT_REGION")]
+    fn mantle_secret_names_include_region_fallbacks(#[case] env_name: &str) {
+        assert!(
+            BEDROCK_MANTLE_MESSAGES_CONFIG
+                .secret_names()
+                .contains(&env_name)
+        );
+    }
+
+    #[rstest]
+    #[case::explicit(Some("explicit"), "explicit")]
+    #[case::environment(None, "mantle")]
+    fn bearer_credentials_replace_forwarded_auth(
+        #[case] key: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
+            .validate_environment(
+                vec![
+                    ("X-Api-Key".into(), "stale".into()),
+                    ("Authorization".into(), "stale".into()),
+                ],
+                key,
+                "claude-test",
+                &LitellmParams::default(),
+                &|name| match name {
+                    "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),
+                    "AWS_BEARER_TOKEN_BEDROCK" => Some("aws".into()),
+                    _ => None,
+                },
+            )
+            .unwrap();
+        assert!(environment.headers.is_empty());
+        assert!(
+            matches!(environment.auth, AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret } if secret.expose() == expected)
+        );
+    }
+
+    #[rstest]
+    #[case::url_injection("us-east-1.attacker.test/steal?")]
+    #[case::uppercase("US-EAST-1")]
+    fn invalid_regions_are_rejected_even_with_bearer_auth(#[case] region: &str) {
+        let params = LitellmParams {
+            aws: AwsParams {
+                aws_region_name: Some(region.into()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(matches!(
+            BEDROCK_MANTLE_MESSAGES_CONFIG.get_complete_url(
+                None,
+                "claude-test",
+                &params,
+                false,
+                &|_| None
+            ),
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    #[case::arn("arn:aws:bedrock:us-east-1.attacker.test:123:model/claude-test", None)]
+    #[case::environment("claude-test", Some("us-east-1/steal"))]
+    fn model_arn_and_environment_regions_are_validated(
+        #[case] model: &str,
+        #[case] region: Option<&str>,
+    ) {
+        let url = BEDROCK_MANTLE_MESSAGES_CONFIG.get_complete_url(
+            None,
+            model,
+            &LitellmParams::default(),
+            false,
+            &|name| {
+                (name == "BEDROCK_MANTLE_REGION")
+                    .then(|| region.map(str::to_string))
+                    .flatten()
+            },
+        );
+        assert!(matches!(url, Err(Error::InvalidRequest(_))));
+    }
+
+    #[rstest]
+    fn caller_betas_are_filtered_renamed_and_merged_with_feature_betas() {
+        let request: MessagesRequest = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": [{"type": "dangerous_tool_use"}]})).unwrap();
+        let headers = BEDROCK_MANTLE_MESSAGES_CONFIG.request_headers(
+            vec![(
+                "Anthropic-Beta".into(),
+                "advanced-tool-use-2025-11-20,code-execution-2025-08-25,custom-beta".into(),
+            )],
+            &request,
+        );
+        let actual: std::collections::BTreeSet<&str> = header_value(&headers, "anthropic-beta")
+            .unwrap()
+            .split(',')
+            .collect();
+        let tool_search = AnthropicBeta::ToolSearchTool20251019.to_string();
+        let safeguards = AnthropicBeta::DangerousToolUse20260903.to_string();
+        assert_eq!(
+            actual,
+            ["custom-beta", tool_search.as_str(), safeguards.as_str()]
+                .into_iter()
+                .collect()
+        );
+    }
+
+    #[rstest]
+    #[case::body_only(
+        json!(["custom-beta", "advanced-tool-use-2025-11-20"]),
+        vec![],
+        vec![
+            AnthropicBeta::Other("custom-beta".into()),
+            AnthropicBeta::ToolSearchTool20251019,
+        ]
+    )]
+    #[case::header_and_body(
+        json!(["custom-b"]),
+        vec![("Anthropic-Beta".into(), "custom-a".into())],
+        vec![
+            AnthropicBeta::Other("custom-a".into()),
+            AnthropicBeta::Other("custom-b".into()),
+        ]
+    )]
+    fn body_betas_are_filtered_renamed_and_merged_with_headers(
+        #[case] body_betas: Value,
+        #[case] headers: Headers,
+        #[case] expected: Vec<AnthropicBeta>,
+    ) {
+        let request: MessagesRequest = serde_json::from_value(json!({
+            "model": "claude-test",
+            "max_tokens": 16,
+            "messages": [],
+            "anthropic_beta": body_betas
+        }))
+        .unwrap();
+        let headers = BEDROCK_MANTLE_MESSAGES_CONFIG.request_headers(headers, &request);
+        let actual: BetaSet = header_value(&headers, "anthropic-beta")
+            .unwrap()
+            .parse()
+            .unwrap();
+        let expected: BetaSet = expected.into_iter().collect();
+        assert_eq!(actual, expected);
+    }
+
+    #[rstest]
+    fn wire_body_keeps_streaming_and_removes_sdk_fields() {
+        assert_eq!(BEDROCK_MANTLE_MESSAGES_CONFIG.wire_body(json!({"model": "mantle/us-west-2/anthropic.claude-test", "stream": true, "anthropic_version": "sdk", "anthropic_beta": ["sdk"], "messages": []})), json!({"model": "anthropic.claude-test", "stream": true, "messages": []}));
+    }
+}

litellm-rust/crates/llms/src/bedrock_mantle/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/llms/src/lib.rs

@@ -3,13 +3,17 @@ pub mod aws_textract;
 pub mod azure_ai;
 pub mod base_llm;
 pub mod bedrock;
+pub mod bedrock_mantle;
 pub mod cohere;
 pub mod deepseek;
 mod error;
+pub mod minimax;
 pub mod mistral;
 pub mod openai;
 pub mod openai_like;
+pub mod openrouter;
 pub mod reducto;
+pub mod tencent;
 pub mod vertex_ai;
 
 pub use error::{Error, ErrorDetail, ErrorSource};

litellm-rust/crates/llms/src/minimax/messages/AGENTS.md

@@ -0,0 +1,9 @@
+# rules
+
+- Reuse Anthropic Messages shaping and reasoning translation
+- Use MiniMax credentials and endpoints, and strip billing system blocks
+
+# references
+
+- Python counterpart `litellm/llms/minimax/messages/transformation.py`
+- Upstream contract in `llms-types/src/providers/minimax/AGENTS.md`

litellm-rust/crates/llms/src/minimax/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/minimax/messages/transformation.rs

@@ -0,0 +1,195 @@
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_llms_types::{
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
+    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+};
+use litellm_router_types::LitellmParams;
+
+use crate::{
+    Error,
+    anthropic::{
+        common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
+        messages::{
+            handler::shape_anthropic_messages_request,
+            transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+        },
+    },
+    base_llm::{
+        auth::{AuthScheme, Headers, ValidatedEnvironment},
+        messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
+    },
+};
+
+const API_KEY_ENV: &str = "MINIMAX_API_KEY";
+const API_BASE_ENV: &str = "MINIMAX_API_BASE";
+const DEFAULT_BASE: &str = "https://api.minimax.io/anthropic";
+
+pub struct MinimaxMessagesConfig;
+pub const MINIMAX_MESSAGES_CONFIG: MinimaxMessagesConfig = MinimaxMessagesConfig;
+
+impl BaseMessagesConfig for MinimaxMessagesConfig {
+    fn shape_request(
+        &self,
+        request: MessagesRequest,
+        reasoning_auto_summary: bool,
+    ) -> Result<MessagesRequest, Error> {
+        shape_anthropic_messages_request(request, reasoning_auto_summary)
+    }
+
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        if has_anthropic_credential(&headers) {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
+            litellm_auth::Error::MissingApiKey {
+                provider: "MiniMax",
+                environment_variable: API_KEY_ENV,
+            },
+        ))?;
+        Ok(ValidatedEnvironment {
+            headers,
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Header("x-api-key"),
+                secret: SecretValue::new(key),
+            },
+        })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let base = resolve_non_empty(api_base, env, &[API_BASE_ENV])
+            .unwrap_or_else(|| DEFAULT_BASE.into());
+        let base = base.trim_end_matches('/');
+        if base.ends_with(MESSAGES_PATH) {
+            return Ok(base.into());
+        }
+        Ok(format!("{base}{MESSAGES_PATH}"))
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        let request = transform_messages_request(request, context)?;
+        Ok(MessagesRequest {
+            params: MessagesOptionalParams {
+                system: request
+                    .params
+                    .system
+                    .and_then(filter_billing_headers_from_system),
+                ..request.params
+            },
+            ..request
+        })
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        &[API_KEY_ENV, API_BASE_ENV]
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        update_headers_with_anthropic_beta(headers, request)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn no_env(_: &str) -> Option<String> {
+        None
+    }
+
+    #[rstest]
+    #[case::root("https://proxy.test", "https://proxy.test/v1/messages")]
+    #[case::root_slash("https://proxy.test/", "https://proxy.test/v1/messages")]
+    #[case::full_endpoint("https://proxy.test/v1/messages", "https://proxy.test/v1/messages")]
+    #[case::full_endpoint_slash(
+        "https://proxy.test/v1/messages/",
+        "https://proxy.test/v1/messages"
+    )]
+    #[case::custom_path(
+        "https://proxy.test/anthropic",
+        "https://proxy.test/anthropic/v1/messages"
+    )]
+    fn endpoint_is_appended_once(#[case] base: &str, #[case] expected: &str) {
+        assert_eq!(
+            MINIMAX_MESSAGES_CONFIG.get_complete_url(
+                Some(base),
+                "test-model",
+                &Default::default(),
+                false,
+                &no_env,
+            ),
+            Ok(expected.into())
+        );
+    }
+
+    #[rstest]
+    #[case::explicit(Some("https://explicit.test"), "https://explicit.test/v1/messages")]
+    #[case::environment(None, "https://env.test/v1/messages")]
+    #[case::blank_explicit(Some(" "), "https://env.test/v1/messages")]
+    fn base_precedence(#[case] base: Option<&str>, #[case] expected: &str) {
+        let env =
+            |name: &str| (name == API_BASE_ENV).then(|| "https://env.test/v1/messages/".into());
+        assert_eq!(
+            MINIMAX_MESSAGES_CONFIG.get_complete_url(
+                base,
+                "test-model",
+                &Default::default(),
+                false,
+                &env,
+            ),
+            Ok(expected.into())
+        );
+    }
+
+    #[rstest]
+    #[case::explicit(Some("explicit"), Some("environment"), "explicit")]
+    #[case::environment(None, Some("environment"), "environment")]
+    #[case::blank_explicit(Some(" "), Some("environment"), "environment")]
+    fn key_precedence(
+        #[case] key: Option<&str>,
+        #[case] env_key: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let env = |name: &str| {
+            (name == API_KEY_ENV)
+                .then(|| env_key.map(str::to_owned))
+                .flatten()
+        };
+        let environment = MINIMAX_MESSAGES_CONFIG
+            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .unwrap();
+        assert!(matches!(
+            environment.auth,
+            AuthScheme::Credential {
+                placement: CredentialPlacement::Header("x-api-key"), ref secret
+            } if secret.expose() == expected
+        ));
+    }
+}

litellm-rust/crates/llms/src/minimax/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md

@@ -0,0 +1,10 @@
+# rules
+
+- Send the native Messages payload to `/api/v1/messages` and relay Anthropic SSE
+- Authenticate with a bearer token, never treat `x-api-key` as an OpenRouter credential
+- Keep cache TTLs, routing fields, metadata, and provider usage extensions
+- Strip first-party billing system blocks using the Anthropic helper
+
+# references
+
+- https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages

litellm-rust/crates/llms/src/openrouter/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/openrouter/messages/transformation.rs

@@ -0,0 +1,198 @@
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_http::request::{has_bearer_auth, without_headers};
+use litellm_llms_types::{
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
+    providers::anthropic::DEFAULT_HEADERS,
+};
+use litellm_router_types::LitellmParams;
+
+use crate::{
+    Error,
+    anthropic::{
+        common_utils::filter_billing_headers_from_system,
+        messages::transformation::update_headers_with_anthropic_beta,
+    },
+    base_llm::{
+        auth::{AuthScheme, Headers, ValidatedEnvironment},
+        messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
+    },
+};
+
+const API_KEY_ENV: &str = "OPENROUTER_API_KEY";
+const API_BASE_ENV: &str = "OPENROUTER_API_BASE";
+const DEFAULT_BASE: &str = "https://openrouter.ai/api/v1";
+const MESSAGES_PATH: &str = "/v1/messages";
+
+pub struct OpenRouterAnthropicMessagesConfig;
+pub const OPENROUTER_MESSAGES_CONFIG: OpenRouterAnthropicMessagesConfig =
+    OpenRouterAnthropicMessagesConfig;
+
+impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        let headers = without_headers(headers, &["x-api-key"]);
+        if has_bearer_auth(&headers) {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
+            litellm_auth::Error::MissingApiKey {
+                provider: "OpenRouter",
+                environment_variable: API_KEY_ENV,
+            },
+        ))?;
+        Ok(ValidatedEnvironment {
+            headers: without_headers(headers, &["authorization"]),
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Bearer,
+                secret: SecretValue::new(key),
+            },
+        })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let explicit = api_base
+            .filter(|base| !base.trim().is_empty())
+            .or(params.api_base.as_deref());
+        let base = resolve_non_empty(explicit, env, &[API_BASE_ENV])
+            .unwrap_or_else(|| DEFAULT_BASE.into());
+        let trimmed = base.trim_end_matches('/');
+        if trimmed.ends_with(MESSAGES_PATH) {
+            return Ok(trimmed.into());
+        }
+        Ok(format!(
+            "{}{MESSAGES_PATH}",
+            trimmed.strip_suffix("/v1").unwrap_or(trimmed)
+        ))
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        _context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        Ok(MessagesRequest {
+            params: MessagesOptionalParams {
+                system: request
+                    .params
+                    .system
+                    .and_then(filter_billing_headers_from_system),
+                ..request.params
+            },
+            ..request
+        })
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        &[API_KEY_ENV, API_BASE_ENV]
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        update_headers_with_anthropic_beta(headers, request)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+    use serde_json::json;
+
+    #[rstest]
+    #[case::default(None, "https://openrouter.ai/api/v1/messages")]
+    #[case::root(Some("https://proxy.test"), "https://proxy.test/v1/messages")]
+    #[case::version(
+        Some("https://proxy.test/api/v1/"),
+        "https://proxy.test/api/v1/messages"
+    )]
+    #[case::complete(
+        Some("https://proxy.test/api/v1/messages/"),
+        "https://proxy.test/api/v1/messages"
+    )]
+    fn urls_append_messages_once(#[case] base: Option<&str>, #[case] expected: &str) {
+        assert_eq!(
+            OPENROUTER_MESSAGES_CONFIG
+                .get_complete_url(
+                    base,
+                    "anthropic/test-model",
+                    &LitellmParams::default(),
+                    true,
+                    &|_| None
+                )
+                .unwrap(),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::call(
+        Some("https://call.test/v1"),
+        Some("https://deployment.test/v1"),
+        "https://call.test/v1/messages"
+    )]
+    #[case::deployment(
+        None,
+        Some("https://deployment.test/v1"),
+        "https://deployment.test/v1/messages"
+    )]
+    #[case::blank_call(
+        Some(" "),
+        Some("https://deployment.test/v1"),
+        "https://deployment.test/v1/messages"
+    )]
+    #[case::environment(None, None, "https://secret.test/v1/messages")]
+    fn base_precedence_is_call_deployment_then_secrets(
+        #[case] base: Option<&str>,
+        #[case] deployment: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let params = LitellmParams {
+            api_base: deployment.map(str::to_string),
+            ..Default::default()
+        };
+        assert_eq!(
+            OPENROUTER_MESSAGES_CONFIG
+                .get_complete_url(base, "test-model", &params, false, &|name| (name
+                    == API_BASE_ENV)
+                    .then(|| "https://secret.test/v1".into()))
+                .unwrap(),
+            expected
+        );
+    }
+
+    #[rstest]
+    fn native_payload_is_preserved_except_first_party_billing_blocks() {
+        let input = json!({"model": "anthropic/test-model", "max_tokens": 128, "messages": [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}], "system": [{"type": "text", "text": "x-anthropic-billing-header: private"}, {"type": "text", "text": "instructions"}], "metadata": {"user_id": "user", "custom": "keep"}, "provider": {"order": ["test-provider"]}, "thinking": {"type": "enabled", "budget_tokens": 64}});
+        let request = serde_json::from_value(input.clone()).unwrap();
+        let actual = OPENROUTER_MESSAGES_CONFIG
+            .transform_anthropic_messages_request(request, &MessagesTransformContext::default())
+            .unwrap();
+        let expected = json!({"system": [{"type": "text", "text": "instructions"}]});
+        let body = serde_json::to_value(actual).unwrap();
+        assert_eq!(body["system"], expected["system"]);
+        assert_eq!(body["messages"], input["messages"]);
+        assert_eq!(body["metadata"], input["metadata"]);
+        assert_eq!(body["provider"], input["provider"]);
+        assert_eq!(body["thinking"], input["thinking"]);
+    }
+}

litellm-rust/crates/llms/src/openrouter/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/llms/src/tencent/messages/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- Reuse Anthropic Messages shaping, reasoning translation, and history cleanup
+- Use Tencent credentials and endpoints, and strip billing system blocks
+- Accept a root, versioned base, chat endpoint, or complete Messages endpoint
+
+# references
+
+- Python counterpart `litellm/llms/tencent/messages/transformation.py`
+- https://www.tencentcloud.com/document/product/1300/80632
+- https://cloud.tencent.com/document/product/1823/132246

litellm-rust/crates/llms/src/tencent/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/tencent/messages/transformation.rs

@@ -0,0 +1,241 @@
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_llms_types::{
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
+    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+};
+use litellm_router_types::LitellmParams;
+
+use crate::{
+    Error,
+    anthropic::{
+        common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
+        messages::{
+            handler::shape_anthropic_messages_request,
+            transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+        },
+    },
+    base_llm::{
+        auth::{AuthScheme, Headers, ValidatedEnvironment},
+        messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
+    },
+};
+
+const API_KEY_ENV: &str = "TENCENT_API_KEY";
+const API_BASE_ENV: &str = "TENCENT_API_BASE";
+const ANTHROPIC_API_BASE_ENV: &str = "TENCENT_ANTHROPIC_API_BASE";
+const DEFAULT_BASE: &str = "https://tokenhub-intl.tencentcloudmaas.com";
+
+pub struct TencentAnthropicMessagesConfig;
+pub const TENCENT_MESSAGES_CONFIG: TencentAnthropicMessagesConfig = TencentAnthropicMessagesConfig;
+
+impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
+    fn shape_request(
+        &self,
+        request: MessagesRequest,
+        reasoning_auto_summary: bool,
+    ) -> Result<MessagesRequest, Error> {
+        shape_anthropic_messages_request(request, reasoning_auto_summary)
+    }
+
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        if has_anthropic_credential(&headers) {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
+            litellm_auth::Error::MissingApiKey {
+                provider: "Tencent",
+                environment_variable: API_KEY_ENV,
+            },
+        ))?;
+        Ok(ValidatedEnvironment {
+            headers,
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Header("x-api-key"),
+                secret: SecretValue::new(key),
+            },
+        })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let base = resolve_non_empty(api_base, env, &[ANTHROPIC_API_BASE_ENV, API_BASE_ENV])
+            .unwrap_or_else(|| DEFAULT_BASE.into());
+        let base = base.trim_end_matches('/');
+        if base.ends_with(MESSAGES_PATH) {
+            return Ok(base.into());
+        }
+        let base = base
+            .strip_suffix("/v1/chat/completions")
+            .or_else(|| base.strip_suffix("/v1"))
+            .unwrap_or(base);
+        Ok(format!("{base}{MESSAGES_PATH}"))
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        let request = transform_messages_request(request, context)?;
+        Ok(MessagesRequest {
+            params: MessagesOptionalParams {
+                system: request
+                    .params
+                    .system
+                    .and_then(filter_billing_headers_from_system),
+                ..request.params
+            },
+            ..request
+        })
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        &[API_KEY_ENV, ANTHROPIC_API_BASE_ENV, API_BASE_ENV]
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        update_headers_with_anthropic_beta(headers, request)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn no_env(_: &str) -> Option<String> {
+        None
+    }
+
+    #[rstest]
+    #[case::root("https://proxy.test", "https://proxy.test/v1/messages")]
+    #[case::root_slash("https://proxy.test/", "https://proxy.test/v1/messages")]
+    #[case::full_endpoint("https://proxy.test/v1/messages", "https://proxy.test/v1/messages")]
+    #[case::full_endpoint_slash(
+        "https://proxy.test/v1/messages/",
+        "https://proxy.test/v1/messages"
+    )]
+    #[case::custom_path(
+        "https://proxy.test/anthropic",
+        "https://proxy.test/anthropic/v1/messages"
+    )]
+    #[case::versioned("https://proxy.test/v1", "https://proxy.test/v1/messages")]
+    #[case::chat_endpoint(
+        "https://proxy.test/v1/chat/completions/",
+        "https://proxy.test/v1/messages"
+    )]
+    fn endpoint_is_appended_once(#[case] base: &str, #[case] expected: &str) {
+        assert_eq!(
+            TENCENT_MESSAGES_CONFIG.get_complete_url(
+                Some(base),
+                "test-model",
+                &Default::default(),
+                false,
+                &no_env,
+            ),
+            Ok(expected.into())
+        );
+    }
+
+    #[rstest]
+    #[case::default(None, None, None, &format!("{DEFAULT_BASE}{MESSAGES_PATH}"))]
+    #[case::explicit(
+        Some("https://explicit.test"),
+        Some("https://anthropic.test"),
+        Some("https://env.test"),
+        "https://explicit.test/v1/messages"
+    )]
+    #[case::anthropic_base(
+        None,
+        Some("https://anthropic.test"),
+        Some("https://env.test"),
+        "https://anthropic.test/v1/messages"
+    )]
+    #[case::generic_base(
+        None,
+        None,
+        Some("https://env.test/v1"),
+        "https://env.test/v1/messages"
+    )]
+    #[case::blank_anthropic_base(
+        None,
+        Some(" "),
+        Some("https://env.test"),
+        "https://env.test/v1/messages"
+    )]
+    #[case::blank_explicit(
+        Some(" "),
+        Some("https://anthropic.test/v1/messages/"),
+        None,
+        "https://anthropic.test/v1/messages"
+    )]
+    fn base_precedence(
+        #[case] base: Option<&str>,
+        #[case] anthropic_base: Option<&str>,
+        #[case] generic_base: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let env = |name: &str| match name {
+            ANTHROPIC_API_BASE_ENV => anthropic_base.map(str::to_owned),
+            API_BASE_ENV => generic_base.map(str::to_owned),
+            _ => None,
+        };
+        assert_eq!(
+            TENCENT_MESSAGES_CONFIG.get_complete_url(
+                base,
+                "test-model",
+                &Default::default(),
+                false,
+                &env,
+            ),
+            Ok(expected.into())
+        );
+    }
+
+    #[rstest]
+    #[case::explicit(Some("explicit"), Some("environment"), "explicit")]
+    #[case::environment(None, Some("environment"), "environment")]
+    #[case::blank_explicit(Some(" "), Some("environment"), "environment")]
+    fn key_precedence(
+        #[case] key: Option<&str>,
+        #[case] env_key: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let env = |name: &str| {
+            (name == API_KEY_ENV)
+                .then(|| env_key.map(str::to_owned))
+                .flatten()
+        };
+        let environment = TENCENT_MESSAGES_CONFIG
+            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .unwrap();
+        assert!(matches!(
+            environment.auth,
+            AuthScheme::Credential {
+                placement: CredentialPlacement::Header("x-api-key"), ref secret
+            } if secret.expose() == expected
+        ));
+    }
+}

litellm-rust/crates/llms/src/tencent/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/python-bridge/src/cache/AGENTS.md

@@ -2,6 +2,6 @@
 
 This folder owns how Rust inference reaches the selected cache: global cache selection, route admission, delegation to a Python cache and the experimental V2 native handles. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates
 
-`mod.rs` exposes the cache boundary to routes and module registration. Adapter directories remain private. `selection.rs` owns global cache selection, route admission and inference protocol composition for both adapters
+`mod.rs` exposes the cache boundary to routes and module registration. Adapter directories remain private. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache, and owns route admission and inference protocol composition for both adapters
 
 `python/` delegates operations to the selected Python cache without discovering configuration. `native/` owns the experimental V2 native cache handles. Neither adapter depends on shared selection or the other adapter. Shared composition depends on the adapters, and routes use only the parent module's exports

litellm-rust/crates/python-bridge/src/cache/selection.rs

@@ -9,6 +9,8 @@ use litellm_host::{
 use pyo3::{prelude::*, types::PyDict};
 use std::sync::Arc;
 
+const CACHE_HOST_MODULE: &str = "litellm.rust_bridge.host.cache";
+
 pub(crate) struct Cached<P>(std::marker::PhantomData<P>);
 
 impl<P: Protocol> Protocol for Cached<P> {
@@ -56,7 +58,10 @@ fn selected_cache<'py>(
     kwargs: &Bound<'py, PyDict>,
     call_type: &str,
 ) -> PyResult<Option<Bound<'py, PyAny>>> {
-    let configured = py.import("litellm")?.getattr("cache")?;
+    let configured = py
+        .import(CACHE_HOST_MODULE)?
+        .getattr("configured_cache")?
+        .call0()?;
     if configured.is_none()
         || kwargs
             .get_item("caching")?

litellm-rust/crates/python-bridge/src/marshal.rs

@@ -9,6 +9,8 @@ use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::{Map, Value};
 
+const TIMEOUTS_MODULE: &str = "litellm.rust_bridge.host.timeouts";
+
 /// The keyword arguments every value route shares, validated at the Python boundary.
 pub(crate) struct RouteOptions {
     pub(crate) model: String,
@@ -47,7 +49,7 @@ pub(crate) fn optional_timeout(timeout_seconds: Option<f64>) -> Option<Duration>
 }
 
 pub(crate) fn python_timeout_seconds(py: Python<'_>, timeout: Py<PyAny>) -> PyResult<Option<f64>> {
-    py.import("litellm.rust_bridge.timeouts")?
+    py.import(TIMEOUTS_MODULE)?
         .getattr("timeout_to_seconds")?
         .call1((timeout,))?
         .extract()

litellm-rust/crates/python-bridge/src/python_settings.rs

@@ -3,7 +3,7 @@ use strum::IntoStaticStr;
 
 use crate::coercion::{FieldSpec, ProjectionError};
 
-const MODULE: &str = "litellm.rust_bridge.settings";
+const MODULE: &str = "litellm.rust_bridge.host.settings";
 
 #[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq)]
 pub(crate) enum PythonSettings {
@@ -220,22 +220,26 @@ import sys
 import types
 previous_modules = {
     name: sys.modules[name]
-    for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.settings')
+    for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.settings')
     if name in sys.modules
 }
 litellm = types.ModuleType('litellm')
 litellm.__path__ = []
 rust_bridge = types.ModuleType('litellm.rust_bridge')
 rust_bridge.__path__ = []
-settings = types.ModuleType('litellm.rust_bridge.settings')
+host = types.ModuleType('litellm.rust_bridge.host')
+host.__path__ = []
+settings = types.ModuleType('litellm.rust_bridge.host.settings')
 def http_settings():
     raise ModuleNotFoundError('No module named certifi', name='certifi')
 settings.http_settings = http_settings
 litellm.rust_bridge = rust_bridge
-rust_bridge.settings = settings
+rust_bridge.host = host
+host.settings = settings
 sys.modules['litellm'] = litellm
 sys.modules['litellm.rust_bridge'] = rust_bridge
-sys.modules['litellm.rust_bridge.settings'] = settings
+sys.modules['litellm.rust_bridge.host'] = host
+sys.modules['litellm.rust_bridge.host.settings'] = settings
 ",
                 Some(&locals),
                 Some(&locals),
@@ -257,7 +261,7 @@ sys.modules['litellm.rust_bridge.settings'] = settings
             );
             py.run(
                 c"
-for name in ('litellm.rust_bridge.settings', 'litellm.rust_bridge', 'litellm'):
+for name in ('litellm.rust_bridge.host.settings', 'litellm.rust_bridge.host', 'litellm.rust_bridge', 'litellm'):
     sys.modules.pop(name, None)
 sys.modules.update(previous_modules)
 ",
@@ -279,22 +283,26 @@ import sys
 import types
 previous_modules = {
     name: sys.modules[name]
-    for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.settings')
+    for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.settings')
     if name in sys.modules
 }
 litellm = types.ModuleType('litellm')
 litellm.__path__ = []
 rust_bridge = types.ModuleType('litellm.rust_bridge')
 rust_bridge.__path__ = []
-settings = types.ModuleType('litellm.rust_bridge.settings')
+host = types.ModuleType('litellm.rust_bridge.host')
+host.__path__ = []
+settings = types.ModuleType('litellm.rust_bridge.host.settings')
 def http_settings():
     raise ImportError('cannot import name setting')
 settings.http_settings = http_settings
 litellm.rust_bridge = rust_bridge
-rust_bridge.settings = settings
+rust_bridge.host = host
+host.settings = settings
 sys.modules['litellm'] = litellm
 sys.modules['litellm.rust_bridge'] = rust_bridge
-sys.modules['litellm.rust_bridge.settings'] = settings
+sys.modules['litellm.rust_bridge.host'] = host
+sys.modules['litellm.rust_bridge.host.settings'] = settings
 ",
                 Some(&locals),
                 Some(&locals),
@@ -308,7 +316,7 @@ sys.modules['litellm.rust_bridge.settings'] = settings
             assert_eq!(error.to_string(), "ImportError: cannot import name setting");
             py.run(
                 c"
-for name in ('litellm.rust_bridge.settings', 'litellm.rust_bridge', 'litellm'):
+for name in ('litellm.rust_bridge.host.settings', 'litellm.rust_bridge.host', 'litellm.rust_bridge', 'litellm'):
     sys.modules.pop(name, None)
 sys.modules.update(previous_modules)
 ",

litellm-rust/crates/python-bridge/src/routes/messages/host.rs

@@ -214,7 +214,7 @@ impl MessagesPythonHost {
     ) -> PyResult<MessagesShaping> {
         let module = py.import(ROUTE_HOST_MODULE)?;
         let capabilities: MessagesModelCapabilities = from_py(
-            &py.import("litellm.rust_bridge.model_capabilities")?
+            &py.import("litellm.rust_bridge.host.model_capabilities")?
                 .getattr("anthropic_model_capabilities")?
                 .call1((model, custom_llm_provider))?,
         )?;

litellm-rust/crates/python-bridge/src/routes/ocr/project.rs

@@ -368,11 +368,11 @@ mod tests {
             c"
 import sys
 import types
-timeouts = types.ModuleType('litellm.rust_bridge.timeouts')
+timeouts = types.ModuleType('litellm.rust_bridge.host.timeouts')
 timeouts.timeout_to_seconds = lambda timeout: None if timeout is None else float(timeout)
 sys.modules.setdefault('litellm', types.ModuleType('litellm'))
 sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge'))
-sys.modules['litellm.rust_bridge.timeouts'] = timeouts
+sys.modules['litellm.rust_bridge.host.timeouts'] = timeouts
 ",
         );
     }

litellm-rust/crates/python-bridge/src/secrets/callback.rs

@@ -14,8 +14,7 @@ use pyo3::{
 use super::error::{external_error, read_error};
 
 const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
-const ENVIRONMENT_FALLBACK_LOG: &str =
-    "Defaulting to os.environ value for key=%s. An exception occurred - %s.\n\n%s";
+const DIAGNOSTICS_MODULE: &str = "litellm.rust_bridge.host.diagnostics";
 
 /// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
 /// client, or a manually assigned SDK client.
@@ -112,16 +111,9 @@ impl ExternalSecretManager for PythonSecretManager {
 }
 
 fn log_environment_fallback(py: Python<'_>, name: &str, error: &PyErr) -> PyResult<()> {
-    let traceback = py
-        .import("traceback")?
-        .call_method1("format_exception", (error.value(py),))?;
-    let traceback = "".into_pyobject(py)?.call_method1("join", (traceback,))?;
-    py.import("litellm._logging")?
-        .getattr("verbose_logger")?
-        .call_method1(
-            "error",
-            (ENVIRONMENT_FALLBACK_LOG, name, error.value(py), traceback),
-        )?;
+    py.import(DIAGNOSTICS_MODULE)?
+        .getattr("log_environment_fallback")?
+        .call1((name, error.value(py)))?;
     Ok(())
 }
 
@@ -139,7 +131,7 @@ mod tests {
 
     use litellm_host_python::PythonContext;
 
-    use super::{HANDLER_MODULE, PythonSecretManager, python_name};
+    use super::{DIAGNOSTICS_MODULE, HANDLER_MODULE, PythonSecretManager, python_name};
     use crate::secrets::python_error;
 
     /// `sys.modules` is interpreter-global, so tests that install or rely on the handler module
@@ -250,13 +242,24 @@ class Logger:
         self.calls.append(args)
 logging = types.ModuleType('litellm._logging')
 logging.verbose_logger = Logger()
-sys.modules.setdefault('litellm', types.ModuleType('litellm'))
+for module in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'):
+    sys.modules.setdefault(module, types.ModuleType(module))
 sys.modules.setdefault('litellm._logging', logging)
 ",
             None,
             None,
         )
         .unwrap();
+        pyo3::types::PyModule::from_code(
+            py,
+            &std::ffi::CString::new(include_str!(
+                "../../../../../litellm/rust_bridge/host/diagnostics.py"
+            ))
+            .unwrap(),
+            c"diagnostics.py",
+            &std::ffi::CString::new(DIAGNOSTICS_MODULE).unwrap(),
+        )
+        .unwrap();
         py.import("litellm._logging")
             .unwrap()
             .getattr("verbose_logger")

litellm-rust/crates/python-bridge/src/secrets/python.rs

@@ -7,7 +7,9 @@ use pyo3::prelude::*;
 
 use super::error::external_error;
 
-/// Reads each secret through Python's `get_secret_str`, so the configured manager, the key
+const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager";
+
+/// Reads each secret through the host module's `read_secret_str`, so the configured manager, the key
 /// management settings and the environment fallback behave exactly as they do in Python.
 pub(super) struct PythonSecrets {
     get_secret_str: Arc<Py<PyAny>>,
@@ -17,8 +19,8 @@ pub(super) struct PythonSecrets {
 impl PythonSecrets {
     pub(super) fn new(py: Python<'_>) -> PyResult<Self> {
         Ok(Self::reading_with(
-            py.import("litellm.secret_managers.main")?
-                .getattr("get_secret_str")?
+            py.import(SECRET_MANAGER_MODULE)?
+                .getattr("read_secret_str")?
                 .unbind(),
             PythonContext::capture(py)?,
         ))

litellm-rust/crates/python-bridge/src/secrets/runtime.rs

@@ -15,6 +15,8 @@ use pyo3::{
 
 use crate::execution::{run_async_value, run_sync_value};
 
+const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager";
+
 #[derive(Clone, PartialEq)]
 struct Configuration {
     system: KeyManagementSystem,
@@ -101,7 +103,7 @@ impl NativeSecretManager {
             return Ok(Some(native));
         }
         let config = py
-            .import("litellm.rust_bridge.secret_manager")?
+            .import(SECRET_MANAGER_MODULE)?
             .getattr("native_secret_manager_config")?
             .call1((client,))?;
         if config.is_none() {

litellm-rust/crates/python-bridge/src/secrets/vault.rs

@@ -7,6 +7,8 @@ use pyo3::prelude::*;
 
 use super::mutation::{error_value, http_message, json_value};
 
+const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport";
+
 pub(super) fn failure_value(
     py: Python<'_>,
     failure: Failure,
@@ -82,7 +84,7 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P
                 kwargs.set_item("model", "default-model-name")?;
                 kwargs.set_item("llm_provider", "litellm-httpx-handler")?;
                 kwargs.set_item("headers", pyo3::types::PyDict::new(py))?;
-                py.import("litellm")?
+                py.import("litellm.exceptions")?
                     .getattr("Timeout")?
                     .call((), Some(&kwargs))?
                     .str()?
@@ -173,9 +175,9 @@ impl ErrorContext {
         Ok(Self {
             timeout: timeout.map(|value| value.str()?.extract()).transpose()?,
             aiohttp: py
-                .import("litellm.llms.custom_httpx.http_handler")?
-                .getattr("AsyncHTTPHandler")?
-                .call_method0("_should_use_aiohttp_transport")?
+                .import(HTTP_TRANSPORT_MODULE)?
+                .getattr("uses_aiohttp_transport")?
+                .call0()?
                 .extract()?,
         })
     }

litellm/__init__.py

@@ -440,10 +440,7 @@ autorouter_presets_url: str = os.getenv(
     "LITELLM_AUTOROUTER_PRESETS_URL",
     "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json",
 )
-whats_new_url: str = os.getenv(
-    "LITELLM_WHATS_NEW_URL",
-    "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/whats_new.json",
-)
+whats_new_url: str | None = os.getenv("LITELLM_WHATS_NEW_URL")
 suppress_debug_info: bool = False
 dynamodb_table_name: Optional[str] = None
 s3_callback_params: Optional[Dict] = None

litellm/litellm_core_utils/litellm_logging.py

@@ -2507,7 +2507,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.callback_duration_ms += (time.time() - _start) * 1000
         return payload
 
-    def _transform_usage_objects(self, result):
+    def _transform_usage_objects(self, result: object) -> object:
         if isinstance(result, ResponsesAPIResponse):
             result = result.model_copy()
             transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage)
@@ -2545,13 +2545,13 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def _success_handler_helper_fn(
         self,
-        result=None,
-        start_time=None,
-        end_time=None,
-        cache_hit=None,
+        result: object = None,
+        start_time: datetime.datetime | None = None,
+        end_time: datetime.datetime | None = None,
+        cache_hit: bool | None = None,
         standard_logging_object: StandardLoggingPayload | None = None,
         build_logging_payload: bool = True,
-    ):
+    ) -> tuple[datetime.datetime, datetime.datetime, object]:
         try:
             if start_time is None:
                 start_time = self.start_time

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -1,9 +1,10 @@
 import base64
 import time
 from collections.abc import Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass, replace
 from itertools import groupby
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast
 
 from pydantic import BaseModel
 from typing_extensions import ReadOnly, Required
@@ -154,6 +155,15 @@ class _ToolCallChoice(TypedDict, total=False):
 _ToolCallKey: TypeAlias = tuple[int, int]
 
 
+@dataclass(frozen=True, slots=True)
+class _ToolCallAccumulator:
+    id: str | None = None
+    name: str | None = None
+    type: str | None = None
+    custom_name: str | None = None
+    provider_specific_fields: Mapping[str, object] | None = None
+
+
 class _ToolCallChunk(TypedDict):
     choices: ReadOnly[Sequence[_ToolCallChoice]]
 
@@ -503,7 +513,7 @@ class ChunkProcessor:
         self, tool_call_chunks: Sequence["_ToolCallChunk"]
     ) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall]:
         tool_calls_list: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] = []
-        tool_call_map: Final[dict[_ToolCallKey, dict[str, Any]]] = {}
+        tool_call_map: Final[dict[_ToolCallKey, _ToolCallAccumulator]] = {}
 
         for chunk in tool_call_chunks:
             choices = chunk["choices"]
@@ -537,49 +547,43 @@ class ChunkProcessor:
                         index = (choice_index, getattr(tool_call, "index", 0))
 
                     if index not in tool_call_map:
-                        tool_call_map[index] = {
-                            "id": None,
-                            "name": None,
-                            "type": None,
-                            "custom_name": None,
-                            "provider_specific_fields": None,
-                        }
+                        tool_call_map[index] = _ToolCallAccumulator()
 
                     # Extract id, type, and function data (handle both dict and object)
                     if isinstance(tool_call, dict):
                         if fragment_id := tool_call.get("id"):
-                            tool_call_map[index]["id"] = fragment_id
+                            tool_call_map[index] = replace(tool_call_map[index], id=fragment_id)
                         if fragment_type := tool_call.get("type"):
-                            tool_call_map[index]["type"] = fragment_type
+                            tool_call_map[index] = replace(tool_call_map[index], type=fragment_type)
 
                         function = tool_call.get("function", {})
                         if isinstance(function, dict):
                             if fragment_name := function.get("name"):
-                                tool_call_map[index]["name"] = fragment_name
+                                tool_call_map[index] = replace(tool_call_map[index], name=fragment_name)
                         else:
                             # function is an object
                             if function_name := getattr(function, "name", None):
-                                tool_call_map[index]["name"] = function_name
+                                tool_call_map[index] = replace(tool_call_map[index], name=function_name)
 
                         custom = tool_call.get("custom")
                         if isinstance(custom, dict):
                             if custom_name := custom.get("name"):
-                                tool_call_map[index]["custom_name"] = custom_name
+                                tool_call_map[index] = replace(tool_call_map[index], custom_name=custom_name)
                     else:
                         # tool_call is an object
                         if hasattr(tool_call, "id") and tool_call.id:
-                            tool_call_map[index]["id"] = tool_call.id
+                            tool_call_map[index] = replace(tool_call_map[index], id=tool_call.id)
                         if hasattr(tool_call, "type") and tool_call.type:
-                            tool_call_map[index]["type"] = tool_call.type
+                            tool_call_map[index] = replace(tool_call_map[index], type=tool_call.type)
                         if object_function_name := getattr(getattr(tool_call, "function", None), "name", None):
-                            tool_call_map[index]["name"] = object_function_name
+                            tool_call_map[index] = replace(tool_call_map[index], name=object_function_name)
 
                         object_custom: ChatCompletionDeltaCustomToolCallPayload | None = getattr(
                             tool_call, "custom", None
                         )
                         if object_custom is not None:
                             if getattr(object_custom, "name", None):
-                                tool_call_map[index]["custom_name"] = object_custom.name
+                                tool_call_map[index] = replace(tool_call_map[index], custom_name=object_custom.name)
 
                     # Preserve provider_specific_fields from streaming chunks
                     provider_fields: object = None
@@ -602,12 +606,14 @@ class ChunkProcessor:
 
                     if provider_fields:
                         # Merge provider_specific_fields if multiple chunks have them
-                        merged_provider_fields = tool_call_map[index]["provider_specific_fields"]
-                        if merged_provider_fields is None:
-                            merged_provider_fields = {}
-                            tool_call_map[index]["provider_specific_fields"] = merged_provider_fields
-                        if isinstance(provider_fields, dict):
-                            merged_provider_fields.update(provider_fields)
+                        tool_call_map[index] = replace(
+                            tool_call_map[index],
+                            provider_specific_fields=(
+                                {**(tool_call_map[index].provider_specific_fields or {}), **provider_fields}
+                                if isinstance(provider_fields, dict)
+                                else dict(tool_call_map[index].provider_specific_fields or {})
+                            ),
+                        )
 
         joined_fragments: Final = self._join_fragments_by_key_and_field(
             self._iter_tool_call_fragments(tool_call_chunks)
@@ -616,37 +622,37 @@ class ChunkProcessor:
         # Convert the map to a list of tool calls
         for index in sorted(tool_call_map.keys()):
             tool_call_data = tool_call_map[index]
-            if tool_call_data["id"] and tool_call_data["custom_name"]:
+            if tool_call_data.id and tool_call_data.custom_name:
                 tool_calls_list.append(
                     ChatCompletionMessageCustomToolCall(
-                        id=tool_call_data["id"],
+                        id=tool_call_data.id,
                         custom=ChatCompletionCustomToolCallPayload(
-                            name=tool_call_data["custom_name"],
+                            name=tool_call_data.custom_name,
                             input=joined_fragments.get((index, "custom_input"), ""),
                         ),
                     )
                 )
-            elif tool_call_data["id"] and tool_call_data["name"]:
+            elif tool_call_data.id and tool_call_data.name:
                 combined_arguments = joined_fragments.get((index, "arguments"), "") or "{}"
 
                 # Build function - provider_specific_fields should be on tool_call level, not function level
                 function = Function(
                     arguments=combined_arguments,
-                    name=tool_call_data["name"],
+                    name=tool_call_data.name,
                 )
 
                 # Prepare params for ChatCompletionMessageToolCall
-                tool_call_params = {
-                    "id": tool_call_data["id"],
-                    "function": function,
-                    "type": tool_call_data["type"] or "function",
-                }
-
                 # Add provider_specific_fields if present (for thought signatures in Gemini 3)
-                if tool_call_data.get("provider_specific_fields"):
-                    tool_call_params["provider_specific_fields"] = tool_call_data["provider_specific_fields"]
-
-                tool_call = ChatCompletionMessageToolCall(**tool_call_params)
+                tool_call = ChatCompletionMessageToolCall(
+                    id=tool_call_data.id,
+                    function=function,
+                    type=tool_call_data.type or "function",
+                    **(
+                        {"provider_specific_fields": dict(tool_call_data.provider_specific_fields)}
+                        if tool_call_data.provider_specific_fields
+                        else {}
+                    ),
+                )
                 tool_calls_list.append(tool_call)
 
         return tool_calls_list

litellm/llms/bedrock/audio_transcription/__init__.py

@@ -6,8 +6,8 @@ import httpx
 from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
 from litellm.rust_bridge import runtime
 from litellm.rust_bridge.catalog import Route, RouteContext
+from litellm.rust_bridge.host.timeouts import timeout_to_seconds
 from litellm.rust_bridge.public_call import NativeCall
-from litellm.rust_bridge.timeouts import timeout_to_seconds
 from litellm.rust_bridge.transcription.native import (
     NATIVE_ATRANSCRIPTION,
     NATIVE_TRANSCRIPTION,

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -474,7 +474,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         Returns:
             AgentCoreParsedResponse: Parsed response data
         """
-        content_type: Final = raw_response.headers.get("content-type", "").lower()
+        content_type: Final = ", ".join(httpx.Headers(raw_response.headers).get_list("content-type")).lower()
         verbose_logger.debug("AgentCore response Content-Type: %s", content_type)
 
         # Parse response based on content type
@@ -698,7 +698,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         )
 
         # Check if response is JSON (agent used sync return) instead of SSE
-        content_type: Final = response.headers.get("content-type", "").lower()
+        content_type: Final = ", ".join(httpx.Headers(response.headers).get_list("content-type")).lower()
         if "application/json" in content_type:
             verbose_logger.debug(
                 "AgentCore streaming: received JSON response instead of SSE, converting to single-chunk stream"
@@ -920,7 +920,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         )
 
         # Check if response is JSON (agent used sync return) instead of SSE
-        content_type: Final = response.headers.get("content-type", "").lower()
+        content_type: Final = ", ".join(httpx.Headers(response.headers).get_list("content-type")).lower()
         if "application/json" in content_type:
             verbose_logger.debug(
                 "AgentCore streaming: received JSON response instead of SSE, converting to single-chunk stream"

litellm/llms/bedrock_mantle/chat/transformation.py

@@ -128,6 +128,30 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
         )
         return [*base_params, *extra_params]
 
+    def map_openai_params(
+        self,
+        non_default_params: dict[str, object],  # mutable-ok: OpenAILikeChatConfig.map_openai_params signature
+        optional_params: dict[str, object],  # mutable-ok: OpenAILikeChatConfig.map_openai_params signature
+        model: str,
+        drop_params: bool,
+        replace_max_completion_tokens_with_max_tokens: bool = True,
+    ) -> dict[str, object]:  # mutable-ok: OpenAILikeChatConfig.map_openai_params signature
+        gpt_reasoning_model: Final = is_gpt_reasoning_series_name(model)
+        mapped_params: Final[dict[str, object]] = super().map_openai_params(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # OpenAILikeChatConfig types its params and result as bare dicts  # mutable-ok: parent returns a dict
+            non_default_params,
+            optional_params,
+            model,
+            drop_params,
+            replace_max_completion_tokens_with_max_tokens and not gpt_reasoning_model,
+        )
+        if not gpt_reasoning_model or "max_tokens" not in mapped_params:
+            return mapped_params
+        return {
+            param: value
+            for param, value in (("max_completion_tokens", mapped_params["max_tokens"]), *mapped_params.items())
+            if param != "max_tokens"
+        }
+
     def _supports_reasoning(self, model: str) -> bool:
         try:
             return litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider)

litellm/llms/gemini/count_tokens/handler.py

@@ -1,6 +1,9 @@
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, RootModel, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@@ -12,6 +15,16 @@ else:
     GenerateContentContentListUnionDict = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _GeminiCountTokensResponse(TypedDict):
+    totalTokens: NotRequired[ReadOnly[int | float]]
+    tokenizer_used: NotRequired[ReadOnly[str]]
+
+
+class _GeminiCountTokensResponseRoot(RootModel[_GeminiCountTokensResponse]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
 class GoogleAIStudioTokenCounter:
     def _clean_contents_for_gemini_api(self, contents: Any) -> object:
         """
@@ -79,7 +92,7 @@ class GoogleAIStudioTokenCounter:
 
     async def acount_tokens(
         self,
-        contents: Any,
+        contents: Sequence[Mapping[str, object]] | None,
         model: str,
         api_key: str | None = None,
         api_base: str | None = None,
@@ -143,8 +156,8 @@ class GoogleAIStudioTokenCounter:
             response.raise_for_status()
 
             # Parse response
-            result: Final = response.json()
-            return result
+            result: Final = _GeminiCountTokensResponseRoot.model_validate(response.json()).root
+            return dict(result)
 
         except httpx.HTTPStatusError as e:
             error_msg = f"Google Gen AI Studio API error: {e.response.status_code} - {e.response.text}"

litellm/llms/openrouter/image_edit/transformation.py

@@ -41,11 +41,14 @@ Response format:
 """
 
 import base64
+from collections.abc import Mapping, Sequence
 from io import BufferedReader, BytesIO
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
 from httpx._types import RequestFiles
+from pydantic import ConfigDict, RootModel, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm.images.utils import ImageEditRequestUtils
@@ -71,6 +74,52 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _ImageUrl(TypedDict):
+    url: ReadOnly[NotRequired[str | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _Image(TypedDict):
+    image_url: ReadOnly[NotRequired[_ImageUrl]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _Message(TypedDict):
+    images: ReadOnly[NotRequired[Sequence[_Image]]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _Choice(TypedDict):
+    message: ReadOnly[NotRequired[_Message]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _TokensDetails(TypedDict):
+    image_tokens: ReadOnly[NotRequired[int | float]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _Usage(TypedDict):
+    prompt_tokens: ReadOnly[NotRequired[int | float]]
+    total_tokens: ReadOnly[NotRequired[int | float]]
+    completion_tokens_details: ReadOnly[NotRequired[_TokensDetails]]
+    prompt_tokens_details: ReadOnly[NotRequired[_TokensDetails | None]]
+    cost: ReadOnly[NotRequired[int | float | str | None]]
+    cost_details: ReadOnly[NotRequired[Mapping[str, object] | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _OpenRouterImageEditResponse(TypedDict):
+    choices: ReadOnly[NotRequired[Sequence[_Choice]]]
+    usage: ReadOnly[NotRequired[_Usage | None]]
+    model: ReadOnly[NotRequired[object]]
+
+
+class _OpenRouterImageEditResponseRoot(RootModel[_OpenRouterImageEditResponse]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
 class OpenRouterImageEditConfig(BaseImageEditConfig):
     """
     Configuration for OpenRouter image editing via chat completions.
@@ -201,7 +250,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig):
         logging_obj: LiteLLMLoggingObj,
     ) -> ImageResponse:
         try:
-            response_json: Final = raw_response.json()
+            response_json: Final[object] = raw_response.json()
         except Exception as e:
             raise OpenRouterException(
                 message=f"Error parsing OpenRouter response: {e}",
@@ -213,7 +262,8 @@ class OpenRouterImageEditConfig(BaseImageEditConfig):
         model_response.data = []
 
         try:
-            choices: Final = response_json.get("choices", [])
+            payload: Final = _OpenRouterImageEditResponseRoot.model_validate(response_json).root
+            choices: Final = payload.get("choices", [])
 
             for choice in choices:
                 message = choice.get("message", {})
@@ -252,7 +302,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig):
                 headers={},
             )
 
-        self._set_usage_and_cost(model_response, response_json, model)
+        self._set_usage_and_cost(model_response, payload, model)
         return model_response
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
@@ -303,11 +353,11 @@ class OpenRouterImageEditConfig(BaseImageEditConfig):
     def _set_usage_and_cost(
         self,
         model_response: ImageResponse,
-        response_json: dict,
+        payload: _OpenRouterImageEditResponse,
         model: str,
     ) -> None:
         """Extract and set usage and cost information from OpenRouter response."""
-        usage_data: Final = response_json.get("usage", {})
+        usage_data: Final = payload.get("usage", {})
         if usage_data:
             prompt_tokens: Final = usage_data.get("prompt_tokens", 0)
             total_tokens: Final = usage_data.get("total_tokens", 0)
@@ -347,7 +397,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig):
                 )
                 response_cost_details.update(cost_details)
 
-        model_response.hidden_params["model"] = response_json.get("model", model)
+        model_response.hidden_params["model"] = payload.get("model", model)
 
     def _read_image_bytes(self, image: FileTypes) -> bytes:
         """Read raw bytes from various image input types."""

litellm/llms/ragflow/vector_stores/transformation.py

@@ -2,6 +2,8 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, RootModel, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
 from litellm.secret_managers.main import get_secret_str
@@ -22,6 +24,25 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _RAGFlowDataset(TypedDict):
+    id: ReadOnly[NotRequired[str | None]]
+    name: ReadOnly[NotRequired[str | None]]
+    create_time: ReadOnly[NotRequired[int | float | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _RAGFlowCreateResponse(TypedDict):
+    data: ReadOnly[NotRequired[_RAGFlowDataset]]
+
+
+class _RAGFlowCreateResponseRoot(RootModel[_RAGFlowCreateResponse]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
+_RAGFLOW_EMPTY_DATA: Final[_RAGFlowDataset] = {}
+
+
 class RAGFlowVectorStoreConfig(BaseVectorStoreConfig):
     """Vector store configuration for RAGFlow datasets."""
 
@@ -185,7 +206,8 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig):
                     headers=response.headers,
                 )
 
-            data: Final = response_json.get("data", {})
+            create_response: Final = _RAGFlowCreateResponseRoot.model_validate(response_json).root
+            data: Final = create_response.get("data", _RAGFLOW_EMPTY_DATA)
 
             # Extract dataset ID
             dataset_id: Final = data.get("id")

litellm/llms/sagemaker/common_utils.py

@@ -4,7 +4,8 @@ from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import Final
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import ConfigDict, RootModel, TypeAdapter, with_config
+from typing_extensions import ReadOnly, Required, TypedDict
 
 import litellm
 from litellm import verbose_logger
@@ -15,6 +16,38 @@ from litellm.types.utils import StreamingChatCompletionChunk
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 
 
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _SageMakerToken(TypedDict, total=False):
+    text: ReadOnly[str | None]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _SageMakerChunkData(TypedDict, total=False):
+    token: ReadOnly[_SageMakerToken | None]
+    index: ReadOnly[int | None]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _SageMakerMessagesChunkData(TypedDict, total=False):
+    choices: ReadOnly[Required[list[Mapping[str, object]]]]
+
+
+class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
+class _SageMakerMessagesChunkDataRoot(RootModel[_SageMakerMessagesChunkData]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
+def _parse_sagemaker_chunk_data(data: str) -> _SageMakerChunkData:
+    return _SageMakerChunkDataRoot.model_validate(json.loads(data)).root
+
+
+def _parse_sagemaker_messages_chunk_data(data: str) -> _SageMakerMessagesChunkData:
+    return _SageMakerMessagesChunkDataRoot.model_validate(json.loads(data)).root
+
+
 def _load_sagemaker_response_stream_shape():
     try:
         from botocore.loaders import Loader
@@ -62,12 +95,12 @@ class AWSEventStreamDecoder:
         self.content_blocks: list = []
         self.is_messages_api = is_messages_api
 
-    def _chunk_parser_messages_api(self, chunk_data: dict) -> StreamingChatCompletionChunk:
+    def _chunk_parser_messages_api(self, chunk_data: _SageMakerMessagesChunkData) -> StreamingChatCompletionChunk:
         openai_chunk: Final = StreamingChatCompletionChunk(**{"model": self.model, **chunk_data})
 
         return openai_chunk
 
-    def _chunk_parser(self, chunk_data: dict) -> GChunk:
+    def _chunk_parser(self, chunk_data: _SageMakerChunkData) -> GChunk:
         verbose_logger.debug("in sagemaker chunk parser, chunk_data %s", chunk_data)
         _token: Final = chunk_data.get("token", {}) or {}
         _index: Final = chunk_data.get("index", None) or 0
@@ -113,11 +146,12 @@ class AWSEventStreamDecoder:
 
                     # Try to parse the accumulated JSON
                     try:
-                        _data = json.loads(accumulated_json)
                         if self.is_messages_api:
-                            yield self._chunk_parser_messages_api(chunk_data=_data)
+                            yield self._chunk_parser_messages_api(
+                                chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json)
+                            )
                         else:
-                            yield self._chunk_parser(chunk_data=_data)
+                            yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json))
                         # Reset accumulated_json after successful parsing
                         accumulated_json = ""
                     except json.JSONDecodeError:
@@ -127,11 +161,12 @@ class AWSEventStreamDecoder:
         # Handle any remaining data after the iterator is exhausted
         if accumulated_json:
             try:
-                _data = json.loads(accumulated_json)
                 if self.is_messages_api:
-                    yield self._chunk_parser_messages_api(chunk_data=_data)
+                    yield self._chunk_parser_messages_api(
+                        chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json)
+                    )
                 else:
-                    yield self._chunk_parser(chunk_data=_data)
+                    yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json))
             except json.JSONDecodeError:
                 # Handle or log any unparseable data at the end
                 verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json)
@@ -161,11 +196,12 @@ class AWSEventStreamDecoder:
                         accumulated_json += message
 
                         # Try to parse the accumulated JSON
-                        _data = json.loads(accumulated_json)
                         if self.is_messages_api:
-                            yield self._chunk_parser_messages_api(chunk_data=_data)
+                            yield self._chunk_parser_messages_api(
+                                chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json)
+                            )
                         else:
-                            yield self._chunk_parser(chunk_data=_data)
+                            yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json))
                         # Reset accumulated_json after successful parsing
                         accumulated_json = ""
                 except json.JSONDecodeError:
@@ -181,11 +217,12 @@ class AWSEventStreamDecoder:
         # Handle any remaining data after the iterator is exhausted
         if accumulated_json:
             try:
-                _data = json.loads(accumulated_json)
                 if self.is_messages_api:
-                    yield self._chunk_parser_messages_api(chunk_data=_data)
+                    yield self._chunk_parser_messages_api(
+                        chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json)
+                    )
                 else:
-                    yield self._chunk_parser(chunk_data=_data)
+                    yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json))
             except json.JSONDecodeError:
                 # Handle or log any unparseable data at the end
                 verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json)

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -1,6 +1,9 @@
+from collections.abc import Sequence
 from typing import Final, Literal
 
 import httpx
+from pydantic import ConfigDict, RootModel, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -13,10 +16,6 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
 )
 from litellm.llms.openai.openai import AllMessageValues
-from litellm.types.llms.vertex_ai import (
-    CachedContentListAllResponseBody,
-    VertexAICachedContentResponseObject,
-)
 from litellm.utils import is_prompt_caching_valid_prompt
 
 from ..common_utils import VertexAIError, get_vertex_base_url
@@ -32,6 +31,31 @@ local_cache_obj: Final = Cache(type=LiteLLMCacheType.LOCAL)  # only used for cal
 MAX_PAGINATION_PAGES: Final = 100  # Reasonable upper bound for pagination
 
 
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _CachedContentItem(TypedDict):
+    displayName: ReadOnly[NotRequired[object]]
+    name: ReadOnly[NotRequired[str | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _CachedContentListBody(TypedDict):
+    cachedContents: ReadOnly[NotRequired[Sequence[_CachedContentItem]]]
+    nextPageToken: ReadOnly[NotRequired[str | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _CreatedCachedContent(TypedDict):
+    name: ReadOnly[NotRequired[str | None]]
+
+
+class _CachedContentListBodyRoot(RootModel[_CachedContentListBody]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
+class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
 class ContextCachingEndpoints(VertexBase):
     """
     Covers context caching endpoints for Vertex AI + Google AI Studio
@@ -157,25 +181,20 @@ class ContextCachingEndpoints(VertexBase):
             except Exception as e:
                 raise VertexAIError(status_code=500, message=str(e))
 
-            raw_response = resp.json()
+            raw_response = _CachedContentListBodyRoot.model_validate(resp.json()).root
             logging_obj.post_call(original_response=raw_response)
 
             if "cachedContents" not in raw_response:
                 return None
 
-            all_cached_items = CachedContentListAllResponseBody(**raw_response)
-
-            if "cachedContents" not in all_cached_items:
-                return None
-
             # Check current page for matching cache_key
-            for cached_item in all_cached_items["cachedContents"]:
+            for cached_item in raw_response["cachedContents"]:
                 display_name = cached_item.get("displayName")
                 if display_name is not None and display_name == cache_key:
                     return cached_item.get("name")
 
             # Check if there are more pages
-            page_token = all_cached_items.get("nextPageToken")
+            page_token = raw_response.get("nextPageToken")
             if not page_token:
                 # No more pages, cache not found
                 break
@@ -249,25 +268,20 @@ class ContextCachingEndpoints(VertexBase):
             except Exception as e:
                 raise VertexAIError(status_code=500, message=str(e))
 
-            raw_response = resp.json()
+            raw_response = _CachedContentListBodyRoot.model_validate(resp.json()).root
             logging_obj.post_call(original_response=raw_response)
 
             if "cachedContents" not in raw_response:
                 return None
 
-            all_cached_items = CachedContentListAllResponseBody(**raw_response)
-
-            if "cachedContents" not in all_cached_items:
-                return None
-
             # Check current page for matching cache_key
-            for cached_item in all_cached_items["cachedContents"]:
+            for cached_item in raw_response["cachedContents"]:
                 display_name = cached_item.get("displayName")
                 if display_name is not None and display_name == cache_key:
                     return cached_item.get("name")
 
             # Check if there are more pages
-            page_token = all_cached_items.get("nextPageToken")
+            page_token = raw_response.get("nextPageToken")
             if not page_token:
                 # No more pages, cache not found
                 break
@@ -424,14 +438,11 @@ class ContextCachingEndpoints(VertexBase):
         except httpx.TimeoutException:
             raise VertexAIError(status_code=408, message="Timeout error occurred.")
 
-        raw_response_cached: Final = response.json()
-        cached_content_response_obj: Final = VertexAICachedContentResponseObject(
-            name=raw_response_cached.get("name"), model=raw_response_cached.get("model")
-        )
+        raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
         return (
             non_cached_messages,
             optional_params,
-            cached_content_response_obj["name"],
+            raw_response_cached.get("name"),
         )
 
     async def async_check_and_create_cache(
@@ -580,14 +591,11 @@ class ContextCachingEndpoints(VertexBase):
         except httpx.TimeoutException:
             raise VertexAIError(status_code=408, message="Timeout error occurred.")
 
-        raw_response_cached: Final = response.json()
-        cached_content_response_obj: Final = VertexAICachedContentResponseObject(
-            name=raw_response_cached.get("name"), model=raw_response_cached.get("model")
-        )
+        raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
         return (
             non_cached_messages,
             optional_params,
-            cached_content_response_obj["name"],
+            raw_response_cached.get("name"),
         )
 
     def get_cache(self):

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -2209,10 +2209,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     @staticmethod
     def _set_stream_metadata_on_response(
         model_response: Any,
-        grounding_metadata: list[dict],
-        url_context_metadata: list[dict],
-        safety_ratings: list[dict],
-        citation_metadata: list[dict],
+        grounding_metadata: Sequence[Mapping[str, object]],
+        url_context_metadata: Sequence[Mapping[str, object]],
+        safety_ratings: Sequence[Mapping[str, object]],
+        citation_metadata: Sequence[Mapping[str, object]],
     ) -> None:
         setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
         if grounding_metadata:

litellm/model_prices_and_context_window_backup.json

@@ -99,22 +99,22 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 256000,
-        "max_output_tokens": 256000,
-        "max_tokens": 256000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 8e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md"
     },
     "ai21.jamba-1-5-mini-v1:0": {
         "deprecation_date": "2026-11-26",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 256000,
-        "max_output_tokens": 256000,
-        "max_tokens": 256000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 4e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md"
     },
     "ai21.jamba-instruct-v1:0": {
         "input_cost_per_token": 5e-07,
@@ -307,12 +307,12 @@
     "us.writer.palmyra-x5-v1:0": {
         "input_cost_per_token": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md",
         "supports_function_calling": true,
         "supports_pdf_input": true
     },
@@ -331,12 +331,12 @@
     "writer.palmyra-x5-v1:0": {
         "input_cost_per_token": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md",
         "supports_function_calling": true,
         "supports_pdf_input": true
     },
@@ -13247,22 +13247,26 @@
         "supports_response_schema": true
     },
     "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.18e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.2e-06
+        "output_cost_per_token": 4.2e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.6e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 7.2e-07
+        "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/ap-south-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -13485,22 +13489,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.05e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.03e-06
+        "output_cost_per_token": 4.03e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6.9e-07
+        "output_cost_per_token": 6.9e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-north-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -13721,22 +13729,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.86e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.78e-06
+        "output_cost_per_token": 3.78e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6.5e-07
+        "output_cost_per_token": 6.5e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-west-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -13795,22 +13807,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.45e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.55e-06
+        "output_cost_per_token": 4.55e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.9e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 7.8e-07
+        "output_cost_per_token": 7.8e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-west-2/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -13888,34 +13904,38 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 5.2e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.56e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 5.9e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 9.1e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/eu-south-1/minimax.minimax-m2.1": {
@@ -13993,22 +14013,26 @@
         "cache_creation_input_token_cost": 3.75e-06
     },
     "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.45e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 5.88e-06
+        "output_cost_per_token": 5.88e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 1.01e-06
+        "output_cost_per_token": 1.01e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/sa-east-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -14207,52 +14231,60 @@
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/deepseek.v3.2": {
@@ -14779,22 +14811,26 @@
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": {
         "cost_per_second": 0.011,
@@ -14885,34 +14921,38 @@
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/deepseek.v3.2": {
@@ -16488,11 +16528,12 @@
         "input_cost_per_query": 0.002,
         "input_cost_per_token": 0.0,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 32000,
-        "max_output_tokens": 32000,
-        "max_tokens": 32000,
+        "max_input_tokens": 4096,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "rerank",
-        "output_cost_per_token": 0.0
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md"
     },
     "command-a-03-2025": {
         "input_cost_per_token": 2.5e-06,
@@ -23666,10 +23707,11 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -38426,25 +38468,28 @@
         "supports_tool_choice": false
     },
     "meta.llama3-1-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
-        "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "meta.llama3-1-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-8b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -38497,6 +38542,7 @@
         "supports_vision": true
     },
     "meta.llama3-3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -38504,37 +38550,44 @@
         "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-3-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "meta.llama4-maverick-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.4e-07,
         "input_cost_per_token_batches": 1.2e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 9.7e-07,
         "output_cost_per_token_batches": 4.85e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -38547,15 +38600,17 @@
         "supports_tool_choice": false
     },
     "meta.llama4-scout-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.7e-07,
         "input_cost_per_token_batches": 8.5e-08,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 10000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6.6e-07,
         "output_cost_per_token_batches": 3.3e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -39127,25 +39182,26 @@
         "supports_vision": true
     },
     "mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "mistral.mistral-large-2407-v1:0": {
@@ -39183,23 +39239,24 @@
         "input_cost_per_token_batches": 5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "output_cost_per_token_batches": 1.5e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-small.md",
         "supports_function_calling": true
     },
     "mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "mistral.voxtral-mini-3b-2507": {
@@ -46304,25 +46361,28 @@
         "supports_tool_choice": false
     },
     "us.meta.llama3-1-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
-        "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "us.meta.llama3-1-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-8b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -46375,6 +46435,7 @@
         "supports_vision": true
     },
     "us.meta.llama3-3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -46382,19 +46443,22 @@
         "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-3-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "us.meta.llama4-maverick-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.4e-07,
         "input_cost_per_token_batches": 1.2e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 9.7e-07,
         "output_cost_per_token_batches": 4.85e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -46407,15 +46471,17 @@
         "supports_tool_choice": false
     },
     "us.meta.llama4-scout-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.7e-07,
         "input_cost_per_token_batches": 8.5e-08,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 10000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6.6e-07,
         "output_cost_per_token_batches": 3.3e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -46431,11 +46497,11 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -51082,8 +51148,8 @@
         "input_cost_per_token_flex": 3e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 203000,
-        "max_output_tokens": 4000,
-        "max_tokens": 4000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 2.2e-06,
         "output_cost_per_token_flex": 1.1e-06,
@@ -51091,7 +51157,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.html",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.md",
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false
@@ -51120,8 +51186,8 @@
         "input_cost_per_token_flex": 3.5e-08,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 203000,
-        "max_output_tokens": 4000,
-        "max_tokens": 4000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 4e-07,
         "output_cost_per_token_flex": 2e-07,
@@ -51129,7 +51195,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.html",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.md",
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false
@@ -63565,7 +63631,7 @@
     },
     "azure_ai/kimi-k2.7-code": {
         "cache_read_input_token_cost": 1.9e-07,
-        "deprecation_date": "2026-10-03",
+        "deprecation_date": "2027-04-16",
         "input_cost_per_token": 9.5e-07,
         "litellm_provider": "azure_ai",
         "max_input_tokens": 262144,

litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py

@@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal
 
 import httpx
 from fastapi import HTTPException
+from pydantic import ConfigDict, RootModel, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -65,6 +67,55 @@ from litellm.types.utils import (
 
 GUARDRAIL_NAME: Final = "model_armor"
 
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _MatchResult(TypedDict):
+    matchState: NotRequired[ReadOnly[str]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _DeidentifyData(TypedDict):
+    text: NotRequired[ReadOnly[str]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _DeidentifyResult(TypedDict):
+    matchState: NotRequired[ReadOnly[str]]
+    data: NotRequired[ReadOnly[_DeidentifyData]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _SdpResult(TypedDict):
+    inspectResult: NotRequired[ReadOnly[_MatchResult]]
+    deidentifyResult: NotRequired[ReadOnly[_DeidentifyResult]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _Filter(TypedDict):
+    raiFilterResult: NotRequired[ReadOnly[_MatchResult]]
+    piAndJailbreakFilterResult: NotRequired[ReadOnly[_MatchResult]]
+    maliciousUriFilterResult: NotRequired[ReadOnly[_MatchResult]]
+    csamFilterFilterResult: NotRequired[ReadOnly[_MatchResult]]
+    virusScanFilterResult: NotRequired[ReadOnly[_MatchResult]]
+    sdpFilterResult: NotRequired[ReadOnly[_SdpResult | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _SanitizationResult(TypedDict):
+    filterResults: NotRequired[ReadOnly[Mapping[str, _Filter] | Sequence[_Filter] | str | int | float | bool | None]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _ModelArmorResponse(TypedDict):
+    sanitizationResult: NotRequired[ReadOnly[_SanitizationResult]]
+    sanitizedText: NotRequired[ReadOnly[str | None]]
+    text: NotRequired[ReadOnly[str | None]]
+
+
+class _ModelArmorResponseRoot(RootModel[_ModelArmorResponse]):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+
 # Only these carry the finished output; response.created carries an empty body
 _RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
 
@@ -99,10 +150,11 @@ class ModelArmorAPIError(Exception):
 
 _SCANNED_CONTENT_KEYS: Final = frozenset({"text", "sanitizedText", "findings", "maliciousUriMatchedItems"})
 
+
 RedactablePayload = dict | list | str | int | float | bool | None
 
 
-def _redact_scanned_content(payload: RedactablePayload, depth: int = 0) -> RedactablePayload:
+def _redact_scanned_content(payload: object, depth: int = 0) -> object:
     if depth >= DEFAULT_MAX_RECURSE_DEPTH:
         return "[REDACTED]"
     if isinstance(payload, dict):
@@ -223,12 +275,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             return f"Model Armor API error (upstream {status_code})"
         return f"Model Armor API error (upstream {status_code}): {response_text}"
 
-    def _build_block_error_detail(self, message: str, armor_response: RedactablePayload) -> dict:
+    def _build_block_error_detail(self, message: str, armor_response: Mapping[str, object]) -> dict:
         if self.sanitize_error_detail:
             return {"error": message}
         return {"error": message, "model_armor_response": armor_response}
 
-    def _build_logging_response(self, armor_response: RedactablePayload) -> RedactablePayload:
+    def _build_logging_response(self, armor_response: Mapping[str, object]) -> object:
         if self.sanitize_error_detail:
             return _redact_scanned_content(armor_response)
         return armor_response
@@ -286,7 +338,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         request_data: dict | None = None,
         file_bytes: bytes | None = None,
         file_type: str | None = None,
-    ) -> dict:
+    ) -> _ModelArmorResponse:
         """
         Make request to Model Armor API. Supports both text and file prompt sanitization.
         If file_bytes and file_type are provided, file prompt sanitization is performed.
@@ -361,8 +413,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
 
         json_response: Final = response.json()
         if hasattr(json_response, "__await__"):
-            return await json_response
-        return json_response
+            return _ModelArmorResponseRoot.model_validate(await json_response).root
+        return _ModelArmorResponseRoot.model_validate(json_response).root
 
     def sanitize_file_prompt(self, file_bytes: bytes, file_type: str, source: str = "user_prompt") -> dict:
         """
@@ -378,7 +430,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         else:
             return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}}
 
-    def _should_block_content(self, armor_response: Mapping[str, object], allow_sanitization: bool = False) -> bool:
+    def _should_block_content(self, armor_response: _ModelArmorResponse, allow_sanitization: bool = False) -> bool:
         """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult."""
         for filt in self._filter_result_items(armor_response):
             # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before
@@ -404,7 +456,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         # Fallback dict code removed; all cases handled above
         return False
 
-    def _get_sanitized_content(self, armor_response: Mapping[str, Any]) -> str | None:
+    def _get_sanitized_content(self, armor_response: _ModelArmorResponse) -> str | None:
         """
         Get the sanitized content from a Model Armor response, if available.
         Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found.
@@ -434,7 +486,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         return armor_response.get("sanitizedText") or armor_response.get("text")
 
     @staticmethod
-    def _filter_result_items(armor_response: Mapping[str, Any]) -> Sequence[Any]:
+    def _filter_result_items(armor_response: _ModelArmorResponse) -> Sequence[_Filter]:
         """Every filter result in a scan response.
 
         filterResults is a dict of named filters on most templates and a list on some, so both
@@ -447,7 +499,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             return filter_results
         return []
 
-    def _has_deidentify_match(self, armor_response: Mapping[str, object]) -> bool:
+    def _has_deidentify_match(self, armor_response: _ModelArmorResponse) -> bool:
         """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction."""
         for filter_entry in self._filter_result_items(armor_response):
             sdp = filter_entry.get("sdpFilterResult")
@@ -457,7 +509,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
 
     def _resolve_streaming_outcome(
         self,
-        armor_response: Mapping[str, object],
+        armor_response: _ModelArmorResponse,
         assembled_response: object,
         content: str,
     ) -> tuple[bool, str | None]:
@@ -489,7 +541,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         return False, sanitized_content
 
     @staticmethod
-    def _append_armor_response(existing: object, armor_response: Mapping[str, object]) -> object:
+    def _append_armor_response(existing: object, armor_response: object) -> object:
         """Accumulate scan responses so a later text scan does not drop an earlier file scan.
 
         Returns the single response on its own (backward compatible) and a list once a request

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -245,7 +245,7 @@ class _TransactionFactory(Protocol):
 
 
 class _ModelTransactionClient(LiteLLMBaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)
+    model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True, hide_input_in_errors=True)
 
     tx: _TransactionFactory
 
@@ -1856,8 +1856,8 @@ async def delete_team_models(
     Returns the model_ids that were deleted.
     """
     deleted_model_ids: Final[list[str]] = []
-    async with prisma_client.db.tx() as tx_ctx:
-        tx: Final[_TxModelTables] = tx_ctx
+    transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
+    async with transaction_client.tx() as tx:
         for team_id in team_ids:
             rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable)
             model_ids = [row.model_id for row in rows]

litellm/proxy/management_endpoints/ui_sso.py

@@ -41,7 +41,7 @@ if TYPE_CHECKING:
 import jwt
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status
 from fastapi.responses import RedirectResponse
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 
 import litellm
 from litellm._internal_context import with_service_target
@@ -237,6 +237,12 @@ class CliSsoTeamDetail(LiteLLMBaseModel):
 
 
 _CLI_SSO_TEAM_DETAILS_ADAPTER: Final = TypeAdapter(tuple[CliSsoTeamDetail, ...])
+_CLI_SSO_FLOW_JSON_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(
+    dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+_SSO_USERINFO_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object] | None]] = TypeAdapter(
+    dict[str, object] | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
 _TEAMLESS_CLI_SSO_TEAM_DETAIL: Final = CliSsoTeamDetail(team_models=())
 
 
@@ -371,7 +377,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict:
     flow = _read_cli_sso_flow(cache, _get_cli_sso_flow_cache_key(cast(str, login_id)))
     if isinstance(flow, str):
         try:
-            flow = _as_object(json.loads(flow))
+            flow = _CLI_SSO_FLOW_JSON_ADAPTER.validate_python(json.loads(flow))
         except ValueError:
             flow = None
     if not isinstance(flow, dict) or "poll_secret_hash" not in flow:
@@ -4199,7 +4205,7 @@ class SSOAuthenticationHandler:
                 )
                 if resp.status_code == 200:
                     try:
-                        userinfo_raw: Final[dict[str, object] | None] = resp.json()
+                        userinfo_raw: Final = _SSO_USERINFO_RESPONSE_ADAPTER.validate_python(resp.json())
                         if not userinfo_raw:
                             # JSON null (None) or empty dict ({}) — no identity claims.
                             # Treat as failure so id_token fallback can be attempted.

litellm/proxy/proxy_server.py

@@ -7434,7 +7434,7 @@ class ProxyConfig:
             else:
                 raise ValueError("Invalid Key Management System selected")
 
-            from litellm.rust_bridge.secret_manager import capture_secret_manager
+            from litellm.rust_bridge.host.secret_manager import capture_secret_manager
 
             if litellm.secret_manager_client is not previous_client:
                 capture_secret_manager(litellm.secret_manager_client, key_management_system)

litellm/proxy/public_endpoints/public_endpoints.py

@@ -591,8 +591,8 @@ async def _fetch_remote_whats_new(url: str) -> WhatsNewResponse:
     return WhatsNewResponse.model_validate_json(await _fetch_remote_bytes(url))
 
 
-async def _resolve_whats_new(url: str, fetch: Callable[[str], Awaitable[WhatsNewResponse]]) -> WhatsNewResponse:
-    if os.getenv("LITELLM_LOCAL_WHATS_NEW", "").lower() == "true":
+async def _resolve_whats_new(url: str | None, fetch: Callable[[str], Awaitable[WhatsNewResponse]]) -> WhatsNewResponse:
+    if url is None or os.getenv("LITELLM_LOCAL_WHATS_NEW", "").lower() == "true":
         return _load_bundled_whats_new()
     try:
         return await fetch(url)
@@ -611,7 +611,7 @@ class _WhatsNewCache:
 
 
 async def get_whats_new(
-    url: str,
+    url: str | None,
     fetch: Callable[[str], Awaitable[WhatsNewResponse]] = _fetch_remote_whats_new,
 ) -> WhatsNewResponse:
     cached: Final = _WhatsNewCache.launches
@@ -637,10 +637,10 @@ async def get_public_whats_new() -> WhatsNewResponse:
     """
     Return the launches the dashboard Home page shows under What's new.
 
-    Resolved once per process: fetched from ``litellm.whats_new_url`` (override with ``LITELLM_WHATS_NEW_URL``)
-    on the first request, falling back to the list bundled with the package on any failure, so an airgapped
-    proxy serves the bundled list without retrying. Set ``LITELLM_LOCAL_WHATS_NEW=True`` to skip the fetch.
-    A restart picks up a newly published list.
+    Serves the list bundled with this release. Set ``LITELLM_WHATS_NEW_URL`` to fetch a remote list instead,
+    resolved once per process on the first request and falling back to the bundled list on any failure, so an
+    airgapped proxy serves the bundled list without retrying. ``LITELLM_LOCAL_WHATS_NEW=True`` forces the
+    bundled list even when a URL is set. A restart picks up a newly published list.
     """
     return await get_whats_new(url=litellm.whats_new_url)
 

litellm/rust_bridge/AGENTS.md

@@ -2,6 +2,8 @@
 
 This package owns native rollout and fallback selection, Python public API compatibility, settings projection and the Python bindings supplied to the Rust bridge. Rust core owns provider execution. `litellm-host-python` owns CPython runtime mechanics. `callbacks-legacy-python` owns legacy callback sharing and dispatch policy
 
+`host/` holds the Python modules Rust imports by name to read Python-owned state (provider resolution, model capabilities, settings, caches, diagnostics, legacy callbacks, the secret manager, transport and timeout conversions). Its AGENTS.md states that contract
+
 `lifecycle.py` owns generic inline execution and stream iteration. `streams.py` supplies the product binding and public stream wrappers through the bridge rather than let the generic Rust host import this package by name. Keep one driver implementing `start`, `resume_value`, `resume_error` and idempotent `close`. Do not create a second implementation
 
 Generic execution steps and inline suspension handling must not depend on LiteLLM response metadata. Public stream construction, `_hidden_params` and header compatibility remain product responsibilities. Keep generic stream heads opaque and preserve public header behavior in the product wrappers

litellm/rust_bridge/host/AGENTS.md

@@ -0,0 +1,11 @@
+# Rust host modules
+
+Python state Rust needs but does not own, behind one module per question. Rust imports a module here by path and calls a function by name
+
+- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds)
+- A module is the only place Rust reads that Python state. Rust never does `py.import("litellm").getattr(...)` on a global; it asks here
+- Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers
+- The module path and function signature are a contract. The Rust side pins the path in a `const MODULE` in the crate that asks (`python-bridge/src/routes/messages/host.rs`, `python-bridge/src/secrets/`, `callbacks-legacy-python/src/python.rs`); change both sides together and keep `tests/unit/rust_bridge/host/` green
+- Inputs and outputs use values and frozen snapshots. Retain caller-owned clients and caches as opaque references when identity is required; Rust never discovers them through module globals
+- Moving a question into Rust means replacing the function body with a native call, not adding a second resolver. The Python signature stays
+- A `*/route_host.py` module is not this: it projects one route's kwargs and maps that route's failures. Shared questions belong here

litellm/rust_bridge/host/__init__.py

@@ -0,0 +1 @@
+"""Python modules the Rust bridge imports by name. See AGENTS.md."""

litellm/rust_bridge/host/cache.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from litellm.caching.caching import Cache
+
+
+def configured_cache() -> Cache | None:
+    import litellm
+
+    return litellm.cache

litellm/rust_bridge/host/diagnostics.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import traceback
+
+
+def log_environment_fallback(name: str, error: Exception) -> None:
+    from litellm._logging import verbose_logger
+
+    verbose_logger.error(
+        "Defaulting to os.environ value for key=%s. An exception occurred - %s.\n\n%s",
+        name,
+        error,
+        "".join(traceback.format_exception(error)),
+    )

litellm/rust_bridge/host/http_transport.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+def uses_aiohttp_transport() -> bool:
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+    return AsyncHTTPHandler.should_use_aiohttp_transport()

litellm/rust_bridge/host/secret_manager.py

@@ -157,3 +157,9 @@ def native_secret_manager_config(client: object) -> NativeSecretManagerConfig |
         ):
             return _capture(client, adapter)
     return None
+
+
+def read_secret_str(name: str) -> str | None:
+    from litellm.secret_managers.main import get_secret_str
+
+    return get_secret_str(name)

litellm/rust_bridge/responses/websocket.py

@@ -8,8 +8,8 @@ from typing import Final, Protocol
 import httpx
 from websockets.exceptions import ConnectionClosedOK
 
+from litellm.rust_bridge.host.timeouts import timeout_to_seconds
 from litellm.rust_bridge.loader import get_native_bridge
-from litellm.rust_bridge.timeouts import timeout_to_seconds
 
 
 class RustResponsesWebSocket(Protocol):

model_prices_and_context_window.json

@@ -99,22 +99,22 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 256000,
-        "max_output_tokens": 256000,
-        "max_tokens": 256000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 8e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md"
     },
     "ai21.jamba-1-5-mini-v1:0": {
         "deprecation_date": "2026-11-26",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 256000,
-        "max_output_tokens": 256000,
-        "max_tokens": 256000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 4e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md"
     },
     "ai21.jamba-instruct-v1:0": {
         "input_cost_per_token": 5e-07,
@@ -307,12 +307,12 @@
     "us.writer.palmyra-x5-v1:0": {
         "input_cost_per_token": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md",
         "supports_function_calling": true,
         "supports_pdf_input": true
     },
@@ -331,12 +331,12 @@
     "writer.palmyra-x5-v1:0": {
         "input_cost_per_token": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md",
         "supports_function_calling": true,
         "supports_pdf_input": true
     },
@@ -13247,22 +13247,26 @@
         "supports_response_schema": true
     },
     "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.18e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.2e-06
+        "output_cost_per_token": 4.2e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.6e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 7.2e-07
+        "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/ap-south-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -13485,22 +13489,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.05e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.03e-06
+        "output_cost_per_token": 4.03e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6.9e-07
+        "output_cost_per_token": 6.9e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-north-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -13721,22 +13729,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.86e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.78e-06
+        "output_cost_per_token": 3.78e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6.5e-07
+        "output_cost_per_token": 6.5e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-west-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -13795,22 +13807,26 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.45e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 4.55e-06
+        "output_cost_per_token": 4.55e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3.9e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 7.8e-07
+        "output_cost_per_token": 7.8e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/eu-west-2/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -13888,34 +13904,38 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html"
     },
     "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 5.2e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.56e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 5.9e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 9.1e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/eu-south-1/minimax.minimax-m2.1": {
@@ -13993,22 +14013,26 @@
         "cache_creation_input_token_cost": 3.75e-06
     },
     "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.45e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 5.88e-06
+        "output_cost_per_token": 5.88e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 1.01e-06
+        "output_cost_per_token": 1.01e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/sa-east-1/deepseek.v3.2": {
         "input_cost_per_token": 7.4e-07,
@@ -14207,52 +14231,60 @@
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-east-1/deepseek.v3.2": {
@@ -14779,22 +14811,26 @@
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": {
         "cost_per_second": 0.011,
@@ -14885,34 +14921,38 @@
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "bedrock/us-west-2/deepseek.v3.2": {
@@ -16488,11 +16528,12 @@
         "input_cost_per_query": 0.002,
         "input_cost_per_token": 0.0,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 32000,
-        "max_output_tokens": 32000,
-        "max_tokens": 32000,
+        "max_input_tokens": 4096,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "rerank",
-        "output_cost_per_token": 0.0
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md"
     },
     "command-a-03-2025": {
         "input_cost_per_token": 2.5e-06,
@@ -23666,10 +23707,11 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -38426,25 +38468,28 @@
         "supports_tool_choice": false
     },
     "meta.llama3-1-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
-        "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "meta.llama3-1-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-8b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -38497,6 +38542,7 @@
         "supports_vision": true
     },
     "meta.llama3-3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -38504,37 +38550,44 @@
         "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-3-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 3.5e-06
+        "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md"
     },
     "meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
         "mode": "chat",
-        "output_cost_per_token": 6e-07
+        "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md"
     },
     "meta.llama4-maverick-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.4e-07,
         "input_cost_per_token_batches": 1.2e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 9.7e-07,
         "output_cost_per_token_batches": 4.85e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -38547,15 +38600,17 @@
         "supports_tool_choice": false
     },
     "meta.llama4-scout-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.7e-07,
         "input_cost_per_token_batches": 8.5e-08,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 10000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6.6e-07,
         "output_cost_per_token_batches": 3.3e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -39127,25 +39182,26 @@
         "supports_vision": true
     },
     "mistral.mistral-7b-instruct-v0:2": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md",
         "supports_tool_choice": true
     },
     "mistral.mistral-large-2402-v1:0": {
         "input_cost_per_token": 4e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 1.2e-05,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md",
         "supports_function_calling": true
     },
     "mistral.mistral-large-2407-v1:0": {
@@ -39183,23 +39239,24 @@
         "input_cost_per_token_batches": 5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "output_cost_per_token_batches": 1.5e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-small.md",
         "supports_function_calling": true
     },
     "mistral.mixtral-8x7b-instruct-v0:1": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 32000,
-        "max_output_tokens": 8191,
-        "max_tokens": 8191,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7e-07,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md",
         "supports_tool_choice": true
     },
     "mistral.voxtral-mini-3b-2507": {
@@ -46304,25 +46361,28 @@
         "supports_tool_choice": false
     },
     "us.meta.llama3-1-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
-        "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "us.meta.llama3-1-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.2e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 128000,
-        "max_output_tokens": 2048,
-        "max_tokens": 2048,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 2.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-1-8b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -46375,6 +46435,7 @@
         "supports_vision": true
     },
     "us.meta.llama3-3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 7.2e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -46382,19 +46443,22 @@
         "max_tokens": 4096,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-3-70b-instruct.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
     "us.meta.llama4-maverick-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.4e-07,
         "input_cost_per_token_batches": 1.2e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 9.7e-07,
         "output_cost_per_token_batches": 4.85e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -46407,15 +46471,17 @@
         "supports_tool_choice": false
     },
     "us.meta.llama4-scout-17b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 1.7e-07,
         "input_cost_per_token_batches": 8.5e-08,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_input_tokens": 10000000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6.6e-07,
         "output_cost_per_token_batches": 3.3e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md",
         "supported_modalities": [
             "text",
             "image"
@@ -46431,11 +46497,11 @@
         "input_cost_per_token": 2e-06,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
         "mode": "chat",
         "output_cost_per_token": 6e-06,
-        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md",
         "supports_function_calling": true,
         "supports_tool_choice": false
     },
@@ -51082,8 +51148,8 @@
         "input_cost_per_token_flex": 3e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 203000,
-        "max_output_tokens": 4000,
-        "max_tokens": 4000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 2.2e-06,
         "output_cost_per_token_flex": 1.1e-06,
@@ -51091,7 +51157,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.html",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.md",
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false
@@ -51120,8 +51186,8 @@
         "input_cost_per_token_flex": 3.5e-08,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 203000,
-        "max_output_tokens": 4000,
-        "max_tokens": 4000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 4e-07,
         "output_cost_per_token_flex": 2e-07,
@@ -51129,7 +51195,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.html",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.md",
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false
@@ -63565,7 +63631,7 @@
     },
     "azure_ai/kimi-k2.7-code": {
         "cache_read_input_token_cost": 1.9e-07,
-        "deprecation_date": "2026-10-03",
+        "deprecation_date": "2027-04-16",
         "input_cost_per_token": 9.5e-07,
         "litellm_provider": "azure_ai",
         "max_input_tokens": 262144,

tests/VCR_TESTING.md

@@ -1,4 +1,4 @@
-Unit tests for individual LLM providers.
+Shared VCR documentation for provider tests.
 
 Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI.
 
@@ -25,7 +25,6 @@ The same VCR cache is used by other test directories that exercise live
 provider APIs. The reusable conftest plumbing lives in
 `tests/_vcr_conftest_common.py` and is wired into:
 
-- `tests/llm_translation/`
 - `tests/audio_tests/`
 - `tests/batches_tests/`
 - `tests/guardrails_tests/`
@@ -67,5 +66,5 @@ make test-llm-translation-flush-vcr-cache
 Skip the cache entirely (every call goes live, no recording):
 
 ```bash
-LITELLM_VCR_DISABLE=1 uv run pytest tests/llm_translation/test_<file>.py
+LITELLM_VCR_DISABLE=1 uv run pytest tests/audio_tests/test_<file>.py
 ```

tests/_vcr_conftest_common.py

@@ -1,7 +1,7 @@
 """Shared VCR (Redis-backed) plumbing imported by per-directory conftests.
 
-See ``tests/llm_translation/Readme.md`` for the full design and
-``tests/llm_translation/conftest.py`` for the reference wiring."""
+See ``tests/VCR_TESTING.md`` for the full design and
+``tests/_vcr_conftest_common.py`` for the shared wiring."""
 
 from __future__ import annotations