Merge branch 'BerriAI:main' into fix/prometheus-multiproc-dir-per-boot

.circleci/config.yml

@@ -38,7 +38,7 @@ commands:
     parameters:
       category:
         type: enum
-        enum: ["backend", "client", "redis-compat"]
+        enum: ["backend", "client"]
         default: "backend"
     steps:
       - run:
@@ -339,8 +339,15 @@ commands:
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
   setup_litellm_test_deps:
+    parameters:
+      checkout:
+        type: boolean
+        default: true
     steps:
-      - checkout
+      - when:
+          condition: << parameters.checkout >>
+          steps:
+            - checkout
       - setup_google_dns
       - install_uv
       - install_rust
@@ -376,13 +383,28 @@ jobs:
           shell: bash.exe
           command: bash .circleci/scripts/path_filter.sh windows-release
       - run:
-          name: Install Python
+          name: Install Python 3.11.9 from python.org
+          no_output_timeout: 6m
           command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
+            $ProgressPreference = "SilentlyContinue"
+            $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe"
+            Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120
+            $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde"
+            $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
+            if ($actual -ne $expected) {
+              throw "Python installer hash mismatch: expected $expected got $actual"
+            }
+            $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311"
+            $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru
+            if (-not $process.WaitForExit(300000)) {
+              $process.Kill()
+              throw "Python installer did not finish within 5 minutes"
+            }
+            if ($process.ExitCode -ne 0) {
+              throw "Python installer exited with $($process.ExitCode)"
+            }
+            Remove-Item $installer
+            & C:\Python311\python.exe --version
       - install_windows_toolchain
       - run:
           name: Record the Rust build environment for the release cargo cache key
@@ -405,7 +427,7 @@ jobs:
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
-            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
+            $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             uv build --wheel --out-dir dist
             if ($LASTEXITCODE -ne 0) {
               exit $LASTEXITCODE
@@ -512,11 +534,12 @@ jobs:
               --set image.repository=litellm-ci \
               --set image.tag=${IMAGE_TAG} \
               --set image.pullPolicy=Never
-            # Wait for pod to be ready
-            echo "Waiting 30 seconds for pod to be ready..."
-            sleep 30
+            if ! kubectl rollout status deployment/litellm --timeout=180s; then
+              kubectl describe pods -l app.kubernetes.io/name=litellm
+              kubectl logs -l app.kubernetes.io/name=litellm --tail=200
+              exit 1
+            fi
 
-            # Print pod logs before running tests
             echo "Printing pod logs..."
             kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
 
@@ -1272,7 +1295,7 @@ jobs:
                   echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
-          server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
+          server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
       - start_redis
       - run:
           name: Run owned integration contracts
@@ -1325,12 +1348,27 @@ jobs:
         default: standard
       shards:
         type: integer
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: medium
+    docker:
+      - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    resource_class: small
     working_directory: ~/project
+    environment:
+      UV_PYTHON_DOWNLOADS: never
     steps:
-      - setup_litellm_test_deps
+      - checkout
+      - install_uv
+      - run:
+          name: Install the locked shard verifier and coverage dependencies
+          command: |
+            uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \
+              | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \
+              > /tmp/fan-in-requirements.txt
+            test "$(grep -c '^[^ ]' /tmp/fan-in-requirements.txt)" -eq 6
+            uv venv .venv --python 3.12
+            uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt
       - attach_workspace:
           at: /tmp/integration-shards
       - run:
@@ -1446,7 +1484,8 @@ jobs:
     steps:
       - checkout
       - skip_if_unrelated_changes
-      - setup_litellm_test_deps
+      - setup_litellm_test_deps:
+          checkout: false
       - start_postgres:
           db_name: litellm_test
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
@@ -1491,108 +1530,6 @@ jobs:
       - store_artifacts:
           path: test-results
 
-  mcp_integration:
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    environment:
-      COVERAGE_CORE: sysmon
-      LITELLM_LOCAL_MODEL_COST_MAP: "True"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_litellm_test_deps
-      - run:
-          name: Generate Prisma client
-          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-      - run:
-          name: Install MCP SDK1 peer
-          command: |
-            uv venv --python 3.12 .venv-mcp-peer
-            uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
-            echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
-      - run:
-          name: Run MCP integration tests
-          command: |
-            mkdir -p test-results
-            env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
-              timeout --signal=TERM 20m uv run --no-sync pytest \
-              tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
-              tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
-              --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
-      - install_codecov_cli
-      - run:
-          name: Upload MCP integration coverage
-          when: always
-          command: |
-            [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
-            codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-              -F mcp-integration -C "$CIRCLE_SHA1" \
-              -n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test-results
-
-  redis_compat:
-    parameters:
-      redis_py:
-        type: string
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_if_unrelated_changes:
-          category: redis-compat
-      - setup_litellm_test_deps
-      - run:
-          name: Pin redis-py version
-          command: |
-            uv pip install "redis==<< parameters.redis_py >>"
-            uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
-      - run:
-          name: Install redis-server 7.2.16
-          command: |
-            mkdir -p "$HOME/.local/bin"
-            cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
-            docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
-            docker rm "$cid"
-            redis-server --version | grep -q 'v=7.2.16'
-      - run:
-          name: Run Redis compatibility tests
-          command: |
-            mkdir -p test-results
-            env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
-              timeout --signal=TERM 15m uv run --no-sync pytest \
-              tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
-              tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
-              tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \
-              tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \
-              --tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
-              --cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
-      - when:
-          condition:
-            equal: ["5.3.1", << parameters.redis_py >>]
-          steps:
-            - install_codecov_cli
-            - run:
-                name: Upload Redis compatibility coverage
-                when: always
-                command: |
-                  [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
-                  codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-                    -F redis-compat -C "$CIRCLE_SHA1" \
-                    -n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test-results
-
 workflows:
   migration_startup_scheduled:
     triggers:
@@ -1692,7 +1629,6 @@ workflows:
       - integration_contracts:
           name: integration-mcp
           suite: mcp
-          resource_class: xlarge
           parallelism: 2
           filters: *integration_default_run
       - integration_shard_coverage:
@@ -1749,19 +1685,6 @@ workflows:
           coverage_flag: roi-postgres
           timeout_minutes: 10
           filters: *integration_default_run
-      - mcp_integration:
-          name: mcp-integration
-          filters: *integration_default_run
-      - redis_compat:
-          name: redis-compat-<< matrix.redis_py >>
-          matrix:
-            parameters:
-              redis_py:
-                - "5.3.1"
-                - "6.4.0"
-                - "7.4.1"
-                - "8.0.1"
-          filters: *integration_default_run
       - windows_release_wheel:
           filters: *integration_default_run
       - base_sdk_install:

.circleci/scripts/classify_changes.sh

@@ -14,7 +14,7 @@ outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
-    litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
+    litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock)
       has_redis_compat=true ;;
   esac
   case "$file" in

.circleci/scripts/path_filter.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: path_filter.sh <backend|client|redis-compat>}"
+category="${1:?usage: path_filter.sh <backend|client|windows-release>}"
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 run_full() {

.circleci/scripts/run_integration.sh

@@ -128,20 +128,14 @@ fi
 sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
 awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
 
-isolated_workers=false
 if [ "$mode" = parity ]; then
   export INTEGRATION_WORKERS=1
 else
   case "$suite" in
-    management|accounting) export INTEGRATION_WORKERS=2; isolated_workers=true ;;
-    providers|extensions) export INTEGRATION_WORKERS=4; isolated_workers=true ;;
+    management|accounting) export INTEGRATION_WORKERS=2 ;;
+    providers|extensions) export INTEGRATION_WORKERS=4 ;;
   esac
 fi
-if [ "$isolated_workers" = false ]; then
-  setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
-    .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
-  upstream_pid=$!
-fi
 if [ "$suite" = cost ]; then
   export INTEGRATION_WORKERS=8
 fi
@@ -149,7 +143,16 @@ if [ "$suite" = security ]; then
   export INTEGRATION_WORKERS=4
 fi
 if [ "$suite" = mcp ]; then
-  export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1
+  export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
+fi
+isolated_workers=false
+if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
+  isolated_workers=true
+fi
+if [ "$isolated_workers" = false ]; then
+  setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+    .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
+  upstream_pid=$!
 fi
 coverage_data="$PWD/$results/coverage/data"
 proxy_command=(.venv/bin/python -m integration._support.proxy)
@@ -191,9 +194,6 @@ start_proxy() {
     > "$results/$log_name" 2>&1 &
   launched_pid=$!
 }
-if { [ "$suite" = management ] || [ "$suite" = mcp ]; } && [ "$isolated_workers" = true ]; then
-  export INTEGRATION_PEER_URL=http://127.0.0.1:4001
-fi
 if [ "$isolated_workers" = false ]; then
   start_proxy 4000 proxy.log
   proxy_pid="$launched_pid"
@@ -239,13 +239,24 @@ if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
   shard_arguments=(--shard-files "$results/node-files.txt")
 fi
 
+shared_services=()
+worker_coverage=()
+if [ "$isolated_workers" = false ]; then
+  shared_services=(
+    "REDIS_HOST=$REDIS_HOST" "REDIS_PORT=$REDIS_PORT"
+    "INTEGRATION_PROXY_URL=$INTEGRATION_PROXY_URL" "INTEGRATION_PEER_URL=$INTEGRATION_PEER_URL"
+    "INTEGRATION_UPSTREAM_URL=$INTEGRATION_UPSTREAM_URL" "INTEGRATION_MASTER_KEY=$INTEGRATION_MASTER_KEY"
+  )
+elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
+  worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
+fi
+
 env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
-  DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
-  INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
-  INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
+  DATABASE_URL="$DATABASE_URL" \
+  "${shared_services[@]}" "${worker_coverage[@]}" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
-  INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
+  LITELLM_MODE=PRODUCTION \
   LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -68,7 +68,7 @@ After: the same request comes back with real token counts, so the dashboard show
 - [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
 - [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
-- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
+- [ ] Every automated review on my PR's latest commit has finished, and I have addressed or replied to each of its findings before requesting a maintainer review
 
 ## Delays in PR merge?
 

.github/workflows/test-e2e-redis-chaos.yml

@@ -91,12 +91,12 @@ jobs:
 
       - name: Run the Redis chaos load test
         env:
-          E2E_REDIS_CHAOS: "1"
+          E2E_PERF_TESTS: "1"
           LITELLM_PROXY_URL: http://localhost:4000
           REDIS_HOST: 127.0.0.1
           REDIS_PORT: "6379"
         run: |
-          uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s
+          uv run --no-sync pytest tests/e2e/perf/test_redis_chaos_e2e.py -v --tb=short -rA -s
 
       - name: Show proxy log on failure
         if: failure()

.github/workflows/test-linting.yml

@@ -615,12 +615,6 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
-      - name: check_greptile_files_json
-        run: uv run --no-sync python ./tests/code_coverage_tests/check_greptile_files_json.py
-
-      - name: test_check_greptile_files_json
-        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_greptile_files_json.py
-
       - name: check_no_publicly_known_master_key
         run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
 

.github/workflows/test-unit.yml

@@ -794,6 +794,10 @@ jobs:
               tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
               tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
               tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
+              tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
+              tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
+              tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py
+              tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py
               tests/unit/proxy/_experimental/mcp_server/outbound_credentials
             python-version: "3.12"
             workers: 4
@@ -802,6 +806,28 @@ jobs:
             job-timeout-minutes: 60
             dist: loadscope
 
+          - shard: mcp-server
+            test-path: |-
+              tests/unit/experimental_mcp_client
+              tests/unit/proxy/_experimental/mcp_server
+              tests/unit/responses/mcp
+              --ignore=tests/unit/experimental_mcp_client/test_mcp_client.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
+              --ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials
+            python-version: "3.12"
+            workers: 4
+            reruns: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+            dist: loadscope
+
           - shard: custom-logging
             test-path: |-
               tests/unit/proxy/test_proxy_server_custom_logger.py
@@ -1324,6 +1350,115 @@ jobs:
       - name: Check consumer types with the installed SDK
         if: steps.changes.outputs.decision != 'skip'
         run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json
+  redis-compat:
+    name: redis-py ${{ matrix.redis-py }} compatibility
+    needs: rust-bridge
+    if: ${{ !cancelled() }}
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        redis-py: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
+    env:
+      UV_PYTHON: "3.12"
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      REDIS_PY_VERSION: ${{ matrix.redis-py }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+        with:
+          category: redis-compat
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-uv-downloads
+
+      - name: Set up the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/rust-bridge
+        with:
+          artifact: ${{ needs.rust-bridge.outputs.artifact }}
+
+      - name: Install dependencies with redis-py ${{ matrix.redis-py }}
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 8
+        env:
+          RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
+        run: |
+          if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
+          else
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
+            cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
+          fi
+          uv pip install --python .venv/bin/python "redis==${REDIS_PY_VERSION:?}"
+          uv run --no-sync python -c 'import os, redis; assert redis.__version__ == os.environ["REDIS_PY_VERSION"], redis.__version__'
+
+      - name: Install redis-server 7.2.16
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        run: |
+          mkdir -p "$RUNNER_TEMP/redis-bin"
+          cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
+          docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
+          docker rm "$cid"
+          "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
+          echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
+
+      - name: Run Redis compatibility tests
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 15
+        env:
+          COVERAGE_CORE: sysmon
+        run: |
+          uv run --no-sync pytest \
+            tests/unit/test_redis.py \
+            tests/unit/caching/test_redis_connection_pool.py \
+            tests/unit/caching/test_redis_cluster_cache.py \
+            tests/unit/caching/test_evicted_client_closer.py \
+            tests/integration/sdk/test_redis_cluster_iam_auth.py \
+            --tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \
+            --cov=./litellm --cov-report=xml:coverage.xml
+
+      - name: Upload redis-py ${{ matrix.redis-py }} coverage to Codecov
+        if: always() && steps.changes.outputs.decision != 'skip' && matrix.redis-py == '5.3.1'
+        continue-on-error: true
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
+        with:
+          use_oidc: true
+          files: coverage.xml
+          root_dir: ${{ github.workspace }}
+          flags: redis-compat
+          fail_ci_if_error: false
   coverage:
     name: coverage
     needs: unit
@@ -1369,10 +1504,20 @@ jobs:
           root_dir: ${{ github.workspace }}
           flags: unit
           fail_ci_if_error: false
+
+      - name: Upload MCP server coverage under its own flag
+        continue-on-error: true
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
+        with:
+          use_oidc: true
+          files: coverage-reports/coverage-mcp-server-${{ github.run_id }}-${{ github.run_attempt }}/coverage.xml
+          root_dir: ${{ github.workspace }}
+          flags: mcp-integration
+          fail_ci_if_error: false
   unit-passed:
     name: unit passed
     permissions: {}
-    needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat]
+    needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat, redis-compat]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 2

.github/workflows/weekly_load_anomaly.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Start the proxy
         run: |
-          nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
+          nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
           for _ in $(seq 1 90); do
             if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
               exit 0
@@ -78,7 +78,7 @@ jobs:
         env:
           E2E_WEEKLY_ANOMALY: "1"
         run: |
-          uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
+          uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
 
       - name: Show proxy log on failure
         if: failure()

.greptile/files.json

@@ -1,655 +0,0 @@
-{
-  "files": [
-    {
-      "path": "litellm/_v2/AGENTS.md",
-      "description": "Conventions for code under litellm/_v2/",
-      "scope": [
-        "litellm/_v2/**"
-      ]
-    },
-    {
-      "path": "litellm/_v2/cache/AGENTS.md",
-      "description": "Conventions for code under litellm/_v2/cache/",
-      "scope": [
-        "litellm/_v2/cache/**"
-      ]
-    },
-    {
-      "path": "litellm/proxy/_experimental/mcp_server/AGENTS.md",
-      "description": "Conventions for code under litellm/proxy/_experimental/mcp_server/",
-      "scope": [
-        "litellm/proxy/_experimental/mcp_server/**"
-      ]
-    },
-    {
-      "path": "litellm/rust_bridge/AGENTS.md",
-      "description": "Conventions for code under litellm/rust_bridge/",
-      "scope": [
-        "litellm/rust_bridge/**"
-      ]
-    },
-    {
-      "path": "litellm/rust_bridge/host/AGENTS.md",
-      "description": "Conventions for code under litellm/rust_bridge/host/",
-      "scope": [
-        "litellm/rust_bridge/host/**"
-      ]
-    },
-    {
-      "path": "litellm/tracing/AGENTS.md",
-      "description": "Conventions for code under litellm/tracing/",
-      "scope": [
-        "litellm/tracing/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/",
-      "scope": [
-        "litellm-rust/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/cache-response/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/cache-response/",
-      "scope": [
-        "litellm-rust/crates/cache-response/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/callbacks-legacy-python/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/callbacks-legacy-python/",
-      "scope": [
-        "litellm-rust/crates/callbacks-legacy-python/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/coroutine/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/coroutine/",
-      "scope": [
-        "litellm-rust/crates/coroutine/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/gateway/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/gateway/",
-      "scope": [
-        "litellm-rust/crates/gateway/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/gateway-auth/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/gateway-auth/",
-      "scope": [
-        "litellm-rust/crates/gateway-auth/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/gateway-inference/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/gateway-inference/",
-      "scope": [
-        "litellm-rust/crates/gateway-inference/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/gateway-mcp/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/gateway-mcp/",
-      "scope": [
-        "litellm-rust/crates/gateway-mcp/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/host/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/host/",
-      "scope": [
-        "litellm-rust/crates/host/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/host/src/machine/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/host/src/machine/",
-      "scope": [
-        "litellm-rust/crates/host/src/machine/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/host-http/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/host-http/",
-      "scope": [
-        "litellm-rust/crates/host-http/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/host-native/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/host-native/",
-      "scope": [
-        "litellm-rust/crates/host-native/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/host-python/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/host-python/",
-      "scope": [
-        "litellm-rust/crates/host-python/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/http/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/http/",
-      "scope": [
-        "litellm-rust/crates/http/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference/",
-      "scope": [
-        "litellm-rust/crates/inference/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference-chat/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference-chat/",
-      "scope": [
-        "litellm-rust/crates/inference-chat/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference-messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference-messages/",
-      "scope": [
-        "litellm-rust/crates/inference-messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference-ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference-ocr/",
-      "scope": [
-        "litellm-rust/crates/inference-ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference-responses/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference-responses/",
-      "scope": [
-        "litellm-rust/crates/inference-responses/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/inference-transcription/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/inference-transcription/",
-      "scope": [
-        "litellm-rust/crates/inference-transcription/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/",
-      "scope": [
-        "litellm-rust/crates/llms/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/anthropic/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/",
-      "scope": [
-        "litellm-rust/crates/llms/src/anthropic/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/batches/",
-      "scope": [
-        "litellm-rust/crates/llms/src/anthropic/batches/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/chat/",
-      "scope": [
-        "litellm-rust/crates/llms/src/anthropic/chat/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/count_tokens/",
-      "scope": [
-        "litellm-rust/crates/llms/src/anthropic/count_tokens/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/anthropic/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/aws_textract/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/aws_textract/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/azure_ai/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/azure_ai/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/base_llm/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/base_llm/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/audio_transcription/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock/audio_transcription/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/chat/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock/chat/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock/claude_platform/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/claude_platform/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock/claude_platform/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/bedrock_mantle/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/bedrock_mantle/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/bedrock_mantle/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/cohere/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/cohere/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/deepseek/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/deepseek/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/minimax/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/minimax/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/minimax/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/mistral/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/mistral/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/openai/responses/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/openai/responses/",
-      "scope": [
-        "litellm-rust/crates/llms/src/openai/responses/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/openai_like/chat/",
-      "scope": [
-        "litellm-rust/crates/llms/src/openai_like/chat/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/openrouter/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/openrouter/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/reducto/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/reducto/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/tencent/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/tencent/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/tencent/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/",
-      "scope": [
-        "litellm-rust/crates/llms/src/vertex_ai/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms/src/vertex_ai/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/",
-      "scope": [
-        "litellm-rust/crates/llms-types/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/audio_transcription/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/audio_transcription/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/batches/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/batches/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/chat_completions/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/chat_completions/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/messages/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/messages/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/ocr/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/ocr/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/responses/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/formats/responses/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/anthropic/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/providers/anthropic/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/bedrock/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/providers/bedrock/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/minimax/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/providers/minimax/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/vertex_ai/",
-      "scope": [
-        "litellm-rust/crates/llms-types/src/providers/vertex_ai/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/model-catalog/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/model-catalog/",
-      "scope": [
-        "litellm-rust/crates/model-catalog/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-bridge/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-bridge/",
-      "scope": [
-        "litellm-rust/crates/python-bridge/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-bridge/src/cache/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/",
-      "scope": [
-        "litellm-rust/crates/python-bridge/src/cache/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/native/",
-      "scope": [
-        "litellm-rust/crates/python-bridge/src/cache/native/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/python/",
-      "scope": [
-        "litellm-rust/crates/python-bridge/src/cache/python/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-bridge/src/routes/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-bridge/src/routes/",
-      "scope": [
-        "litellm-rust/crates/python-bridge/src/routes/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/python-compat/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/python-compat/",
-      "scope": [
-        "litellm-rust/crates/python-compat/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/secrets-aws/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/secrets-aws/",
-      "scope": [
-        "litellm-rust/crates/secrets-aws/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/secrets-azure/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/secrets-azure/",
-      "scope": [
-        "litellm-rust/crates/secrets-azure/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/secrets-cyberark/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/secrets-cyberark/",
-      "scope": [
-        "litellm-rust/crates/secrets-cyberark/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/secrets-google/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/secrets-google/",
-      "scope": [
-        "litellm-rust/crates/secrets-google/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/secrets-hashicorp/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/secrets-hashicorp/",
-      "scope": [
-        "litellm-rust/crates/secrets-hashicorp/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/crates/storage-clickhouse/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/crates/storage-clickhouse/",
-      "scope": [
-        "litellm-rust/crates/storage-clickhouse/**"
-      ]
-    },
-    {
-      "path": "litellm-rust/docs/adrs/AGENTS.md",
-      "description": "Conventions for code under litellm-rust/docs/adrs/",
-      "scope": [
-        "litellm-rust/docs/adrs/**"
-      ]
-    },
-    {
-      "path": "tests/AGENTS.md",
-      "description": "Conventions for code under tests/",
-      "scope": [
-        "tests/**"
-      ]
-    },
-    {
-      "path": "tests/e2e/AGENTS.md",
-      "description": "Conventions for code under tests/e2e/",
-      "scope": [
-        "tests/e2e/**"
-      ]
-    },
-    {
-      "path": "tests/e2e_harness/AGENTS.md",
-      "description": "Conventions for code under tests/e2e_harness/",
-      "scope": [
-        "tests/e2e_harness/**"
-      ]
-    },
-    {
-      "path": "tests/integration/AGENTS.md",
-      "description": "Conventions for code under tests/integration/",
-      "scope": [
-        "tests/integration/**"
-      ]
-    },
-    {
-      "path": "tests/integration/translation/AGENTS.md",
-      "description": "Conventions for code under tests/integration/translation/",
-      "scope": [
-        "tests/integration/translation/**"
-      ]
-    },
-    {
-      "path": "tests/rust-python-harness/AGENTS.md",
-      "description": "Conventions for code under tests/rust-python-harness/",
-      "scope": [
-        "tests/rust-python-harness/**"
-      ]
-    },
-    {
-      "path": "tests/rust-python-harness/strategies/e2e_parity/AGENTS.md",
-      "description": "Conventions for code under tests/rust-python-harness/strategies/e2e_parity/",
-      "scope": [
-        "tests/rust-python-harness/strategies/e2e_parity/**"
-      ]
-    },
-    {
-      "path": "tests/rust-python-harness/strategies/trace_parity/AGENTS.md",
-      "description": "Conventions for code under tests/rust-python-harness/strategies/trace_parity/",
-      "scope": [
-        "tests/rust-python-harness/strategies/trace_parity/**"
-      ]
-    },
-    {
-      "path": "tests/rust-python-harness/strategies/unit_tests_parity/AGENTS.md",
-      "description": "Conventions for code under tests/rust-python-harness/strategies/unit_tests_parity/",
-      "scope": [
-        "tests/rust-python-harness/strategies/unit_tests_parity/**"
-      ]
-    },
-    {
-      "path": "tests/rust-python-harness/strategies/unit_tests_rust/AGENTS.md",
-      "description": "Conventions for code under tests/rust-python-harness/strategies/unit_tests_rust/",
-      "scope": [
-        "tests/rust-python-harness/strategies/unit_tests_rust/**"
-      ]
-    },
-    {
-      "path": "tests/test_litellm_rust/AGENTS.md",
-      "description": "Conventions for code under tests/test_litellm_rust/",
-      "scope": [
-        "tests/test_litellm_rust/**"
-      ]
-    },
-    {
-      "path": "tests/unit/AGENTS.md",
-      "description": "Conventions for code under tests/unit/",
-      "scope": [
-        "tests/unit/**"
-      ]
-    },
-    {
-      "path": "tests/unit/rust_bridge/AGENTS.md",
-      "description": "Conventions for code under tests/unit/rust_bridge/",
-      "scope": [
-        "tests/unit/rust_bridge/**"
-      ]
-    },
-    {
-      "path": "ui/litellm-dashboard/AGENTS.md",
-      "description": "Conventions for code under ui/litellm-dashboard/",
-      "scope": [
-        "ui/litellm-dashboard/**"
-      ]
-    },
-    {
-      "path": "ui/litellm-dashboard/src/components/chat/AGENTS.md",
-      "description": "Conventions for code under ui/litellm-dashboard/src/components/chat/",
-      "scope": [
-        "ui/litellm-dashboard/src/components/chat/**"
-      ]
-    }
-  ]
-}

AGENTS.md

@@ -130,4 +130,4 @@ Before implementing:
 
 Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
 
-Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check
+Before requesting maintainer review, verify the current PR tip passes required CI and code coverage and has acceptable reviews from the review bots on the PR, such as Veria and Bugbot. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

README.md

@@ -9,7 +9,7 @@
         <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render" height="40"></a>
         <a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic"><img src="https://railway.com/button.svg" alt="Deploy on Railway" height="40"></a>
         <a href="https://console.aws.amazon.com/cloudshell/home" target="_blank" rel="nofollow"><img src="./.github/deploy-on-aws.png" alt="Deploy on AWS" height="40"></a>
-        <a href="https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true" target="_blank" rel="nofollow"><img src="./.github/deploy-on-gcp.png" alt="Deploy on GCP" height="40"></a>
+        <a href="https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Fterraform-google-litellm&cloudshell_git_branch=v1.105.0&cloudshell_workspace=examples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true" target="_blank" rel="nofollow"><img src="./.github/deploy-on-gcp.png" alt="Deploy on GCP" height="40"></a>
         </p>
     </p>
 <h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://www.litellm.ai/ai-gateway" target="_blank">Website</a></h4>
@@ -342,6 +342,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
 | [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [CoralBricks (`coralbricks`)](https://docs.litellm.ai/docs/providers/coralbricks) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
 | [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
@@ -526,9 +527,9 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra
 
 #### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB
 
-[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
+[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Fterraform-google-litellm&cloudshell_git_branch=v1.105.0&cloudshell_workspace=examples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
 
-Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply.
+Real 1-click. Opens Cloud Shell, clones the [terraform-google-litellm](https://github.com/BerriAI/terraform-google-litellm) module mirror at the latest stable release (a small download instead of the full litellm repo), and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md). Pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply
 
 [Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest)
 

ci_cd/cost_map_guard.py

@@ -56,7 +56,7 @@ def _rendered_schema(cost_map: CostMap) -> str:
 def _evaluation_output_price_failure(
     key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
 ) -> str | None:
-    if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
+    if not isinstance(entry, dict) or entry.get("mode") not in ("evaluation", "decisions"):
         return None
     output_cost: Final = entry.get("output_cost_per_token")
     if (
@@ -67,7 +67,7 @@ def _evaluation_output_price_failure(
     ):
         return None
     return (
-        f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
+        f"{key}: mode evaluation and decisions rows bill input only; output_cost_per_token must be 0 unless the vendor "
         "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
     )
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -1162,7 +1162,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )
         return False
 
-    async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
+    async def check_file_ids_access(self, file_ids: Sequence[str], user_api_key_dict: UserAPIKeyAuth) -> None:
         """
         Check if the user has access to a list of file IDs.
         Only checks managed (unified) file IDs.
@@ -1269,6 +1269,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         elif call_type == CallTypes.acreate_batch.value:
             input_file_id = cast(Optional[str], data.get("input_file_id"))
             if input_file_id:
+                await self.check_file_ids_access((input_file_id,), user_api_key_dict)
                 model_file_id_mapping = await self.get_model_file_id_mapping(
                     [input_file_id], user_api_key_dict.parent_otel_span
                 )
@@ -1315,11 +1316,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             elif retrieve_object_id and accessor_key == "batch_id":
                 await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
         elif call_type == CallTypes.acreate_fine_tuning_job.value:
-            input_file_id = cast(Optional[str], data.get("training_file"))
-            if input_file_id:
-                model_file_id_mapping = await self.get_model_file_id_mapping(
-                    [input_file_id], user_api_key_dict.parent_otel_span
-                )
+            fine_tuning_file_ids: Final = tuple(
+                file_id
+                for file_id in (data.get("training_file"), data.get("validation_file"))
+                if isinstance(file_id, str) and file_id
+            )
+            await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict)
 
         return data
 

litellm-rust/Cargo.lock

@@ -4024,6 +4024,7 @@ dependencies = [
  "litellm-cache-memory",
  "litellm-cache-response",
  "litellm-core-utils",
+ "litellm-framer",
  "litellm-host",
  "litellm-host-native",
  "litellm-http",
@@ -4107,9 +4108,11 @@ name = "litellm-inference-testing"
 version = "0.1.0"
 dependencies = [
  "futures-util",
+ "litellm-auth",
  "litellm-http",
  "litellm-inference",
  "litellm-secrets",
+ "tokio",
 ]
 
 [[package]]

litellm-rust/crates/callbacks-legacy-python/src/adapter.rs

@@ -442,7 +442,7 @@ impl LegacyLogging {
         py: Python<'_>,
         facts: &litellm_host::interceptors::ExecutionFacts,
     ) -> PyResult<HookStep<Self, ()>> {
-        use litellm_host::interceptors::ResultSource;
+        use litellm_host::interceptors::{Cost, ResultSource};
 
         let logger = self.logger()?.object(py);
         let params = logger
@@ -462,6 +462,12 @@ impl LegacyLogging {
             ),
         )?;
         let details = logger.getattr("model_call_details")?;
+        let reported_cost = match &facts.cost {
+            Cost::Reported { amount } => amount.as_f64(),
+            Cost::CacheHit | Cost::Deferred => None,
+        };
+        logger.setattr("_provider_reported_cost", reported_cost)?;
+        details.call_method1("pop", ("response_cost", py.None()))?;
         self.cache_key = match &facts.source {
             ResultSource::Provider => None,
             ResultSource::Cache { key } => Some(key.clone()),
@@ -958,6 +964,81 @@ on_pre_call = lambda additional_args: None
 check = lambda: None
 ";
 
+    #[rstest]
+    #[case::reported(Some(0.37), false)]
+    #[case::free(Some(0.0), false)]
+    #[case::deferred(None, false)]
+    #[case::cached(None, true)]
+    fn result_ready_hands_only_a_reported_cost_to_the_spend_logging_record(
+        #[case] reported: Option<f64>,
+        #[case] cache_hit: bool,
+        #[values(0.001, 7.0)] existing_cost: f64,
+    ) {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = namespace(py, PAYLOAD_LOGGER);
+            run(
+                py,
+                &locals,
+                c"
+logger.litellm_params = {}
+logger.optional_params = {}
+logger.model_call_details = {'response_cost': 7.0}
+logger._provider_reported_cost = None
+",
+            );
+            let mut logging = LegacyLogging {
+                logger: Some(PythonLogger::new(local(&locals, "logger").unbind())),
+                ..legacy_call(py, &locals, false)
+            };
+            let source = if cache_hit {
+                litellm_host::interceptors::ResultSource::Cache {
+                    key: "cached-test".into(),
+                }
+            } else {
+                litellm_host::interceptors::ResultSource::Provider
+            };
+            let facts = litellm_host::interceptors::ExecutionFacts {
+                provider: litellm_host::interceptors::ProviderIdentity {
+                    model: "native-test".into(),
+                    provider: "edenai".into(),
+                },
+                cost: litellm_host::interceptors::Cost::settle(
+                    &source,
+                    reported.and_then(serde_json::Number::from_f64),
+                ),
+                source,
+            };
+            let previous = litellm_host::interceptors::ExecutionFacts {
+                provider: facts.provider.clone(),
+                cost: litellm_host::interceptors::Cost::Reported {
+                    amount: serde_json::Number::from_f64(9.0).unwrap(),
+                },
+                source: litellm_host::interceptors::ResultSource::Provider,
+            };
+            logging.result_ready(py, &previous).unwrap();
+            local(&locals, "logger")
+                .getattr("model_call_details")
+                .unwrap()
+                .set_item("response_cost", existing_cost)
+                .unwrap();
+            let step = logging.result_ready(py, &facts).unwrap();
+            assert!(matches!(step, HookStep::Ready(())));
+            locals.set_item("reported", reported).unwrap();
+            locals.set_item("cache_hit", cache_hit).unwrap();
+            run(
+                py,
+                &locals,
+                c"
+assert logger.model_call_details.get('response_cost') is None, logger.model_call_details
+assert logger._provider_reported_cost == reported, logger._provider_reported_cost
+assert logger.model_call_details['cache_hit'] is cache_hit, logger.model_call_details
+assert logger.update['custom_llm_provider'] == 'edenai', logger.update
+",
+            );
+        });
+    }
+
     const DOCUMENT: &str = "data:application/pdf;base64,YWJj";
     const EDITED: &str = "data:application/pdf;base64,ZWRpdGVk";
 

litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs

@@ -22,6 +22,8 @@ pub enum LlmProviders {
     Cohere,
     #[strum(serialize = "deepseek")]
     Deepseek,
+    #[strum(serialize = "edenai")]
+    Edenai,
     #[strum(serialize = "minimax")]
     Minimax,
     #[strum(serialize = "mistral")]

litellm-rust/crates/host/src/interceptors.rs

@@ -42,10 +42,32 @@ pub enum ResultSource {
     Cache { key: String },
 }
 
+/// What the call cost, settled once by the route. A host records it; it never re-derives it.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Cost {
+    /// The amount the provider itself billed, taken from its response.
+    Reported { amount: serde_json::Number },
+    /// Served from the cache, so nothing was billed.
+    CacheHit,
+    /// No provider figure: the host estimates from usage and its price table.
+    Deferred,
+}
+
+impl Cost {
+    pub fn settle(source: &ResultSource, reported: Option<serde_json::Number>) -> Self {
+        match (source, reported) {
+            (ResultSource::Cache { .. }, _) => Self::CacheHit,
+            (ResultSource::Provider, Some(amount)) => Self::Reported { amount },
+            (ResultSource::Provider, None) => Self::Deferred,
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ExecutionFacts {
     pub provider: ProviderIdentity,
     pub source: ResultSource,
+    pub cost: Cost,
 }
 
 pub trait Interceptors<E>: Send + Sync {

litellm-rust/crates/inference-messages/Cargo.toml

@@ -26,6 +26,7 @@ serde_json = { workspace = true, features = ["preserve_order"] }
 tracing.workspace = true
 
 [dev-dependencies]
+litellm-framer = { workspace = true, features = ["sse"] }
 litellm-cache-memory.workspace = true
 litellm-host-native.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -8,6 +8,7 @@ use litellm_llms::{
     base_llm::messages::transformation::BaseMessagesConfig,
     bedrock::messages::invoke_transformations::anthropic_claude3_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
     deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
+    edenai::messages::transformation::EDENAI_MESSAGES_CONFIG,
     minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG,
     vertex_ai::messages::transformation::VERTEX_ANTHROPIC_MESSAGES_CONFIG,
 };
@@ -53,6 +54,7 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<M
         LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
         LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG,
         LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
+        LlmProviders::Edenai => &EDENAI_MESSAGES_CONFIG,
         LlmProviders::Minimax => &MINIMAX_MESSAGES_CONFIG,
         LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => {
             &VERTEX_ANTHROPIC_MESSAGES_CONFIG
@@ -85,6 +87,7 @@ mod tests {
     #[case::azure_ai("azure_ai")]
     #[case::bedrock("bedrock")]
     #[case::deepseek("deepseek")]
+    #[case::edenai("edenai")]
     #[case::minimax("minimax")]
     #[case::vertex_ai("vertex_ai")]
     fn provider_keeps_its_python_name(#[case] name: &str) {

litellm-rust/crates/inference-messages/src/handler.rs

@@ -25,7 +25,7 @@ use litellm_inference::{context::CallContext, outbound::outbound_request};
 pub(super) struct ProviderCall {
     pub identity: ProviderIdentity,
     pub wire: WireRequest,
-    provider: super::common_utils::MessagesProvider,
+    pub(super) provider: super::common_utils::MessagesProvider,
     signer: Option<litellm_auth_aws::SigV4Signer>,
     timeout: Option<Duration>,
     stream: bool,

litellm-rust/crates/inference-messages/src/lib.rs

@@ -7,7 +7,7 @@ mod types;
 
 use futures_util::FutureExt;
 use litellm_auth::AuthServices;
-use litellm_host::interceptors::{ExecutionFacts, Interceptors, ResultSource};
+use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
 
 use litellm_inference::{caching::CallCache, context::CallContext};
 use litellm_secrets::source::SecretSource;
@@ -87,6 +87,7 @@ impl MessagesRoute {
                 &request.wire,
             );
             let identity = request.identity.clone();
+            let config = request.provider.config();
             let (output, source) = match cache.lookup().await {
                 Some(hit) => hit,
                 None => (
@@ -94,9 +95,14 @@ impl MessagesRoute {
                     ResultSource::Provider,
                 ),
             };
+            let reported = match &output {
+                MessagesCallResponse::Complete(response) => config.reported_cost(response),
+                MessagesCallResponse::Stream { .. } => None,
+            };
             context
                 .result_ready(ExecutionFacts {
                     provider: identity,
+                    cost: Cost::settle(&source, reported),
                     source: source.clone(),
                 })
                 .await?;

litellm-rust/crates/inference-messages/tests/caching.rs

@@ -13,8 +13,8 @@ use litellm_cache_response::{
     CacheOptions, CacheScope, ResponseCache, ResponseCacheConfig, ResponseCacheService, ScopedCache,
 };
 use litellm_host::interceptors::{
-    ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, ResultSource,
-    WireRequest,
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
+    ResultSource, WireRequest,
 };
 use litellm_inference::{
     RouteError,
@@ -241,3 +241,55 @@ async fn messages_cache_identity_follows_resolved_configuration_and_request_call
     first.verify().await;
     second.verify().await;
 }
+
+#[rstest]
+#[case::reported(json!(0.37), Some(0.37))]
+#[case::free(json!(0), Some(0.0))]
+#[case::malformed(json!("invalid"), None)]
+#[case::missing(Value::Null, None)]
+#[tokio::test]
+async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_hit(
+    cache: Arc<dyn ResponseCacheService>,
+    #[case] cost: Value,
+    #[case] expected: Option<f64>,
+) {
+    use litellm_inference_messages::{MessagesCall, MessagesCallResponse};
+    let response = json!({"id": "msg_cost", "type": "message", "role": "assistant", "model": "native-test",
+        "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null,
+        "usage": {"input_tokens": 3, "output_tokens": 2}, "cost": cost});
+    let upstream = support::upstream([support::json_response(response)]).await;
+    let hooks = ChangingHooks::default();
+    let route = support::messages_route(litellm_inference_testing::no_secrets())
+        .with_cache(ScopedCache::new(cache, CacheScope::Shared));
+    let request = || {
+        MessagesCall {
+        body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(),
+        api_key: Some("eden-key".into()), api_base: Some(upstream.uri()), custom_llm_provider: Some("edenai".into()),
+        litellm_params: Default::default(), extra_headers: None, provider_specific_header: None,
+        timeout: None, shaping: Default::default(),
+    }
+    };
+    let first = route.execute(request(), &hooks, None).await.unwrap();
+    let second = route.execute(request(), &hooks, None).await.unwrap();
+    let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) =
+        (first, second)
+    else {
+        panic!("expected Messages responses");
+    };
+    assert_eq!(first, second);
+    assert_eq!(first.extra.get("cost"), Some(&cost));
+    assert_eq!(support::received(&upstream).await.len(), 1);
+    let facts = hooks.facts.lock().unwrap();
+    let [provider, cached] = facts.as_slice() else {
+        panic!("expected provider and cache results");
+    };
+    assert_eq!(provider.source, ResultSource::Provider);
+    assert_eq!(
+        provider.cost,
+        expected.map_or(Cost::Deferred, |amount| Cost::Reported {
+            amount: serde_json::Number::from_f64(amount).unwrap()
+        })
+    );
+    assert!(matches!(cached.source, ResultSource::Cache { .. }));
+    assert_eq!(cached.cost, Cost::CacheHit);
+}

litellm-rust/crates/inference-messages/tests/live.rs

@@ -0,0 +1,8 @@
+#[path = "live/support.rs"]
+mod support;
+
+#[path = "live/edenai.rs"]
+mod edenai;
+
+#[path = "live/vertex_ai.rs"]
+mod vertex_ai;

litellm-rust/crates/inference-messages/tests/live/edenai.rs

@@ -0,0 +1,92 @@
+use litellm_host::interceptors::Cost;
+use litellm_inference_messages::MessagesRoute;
+use litellm_inference_testing::live::within_deadline;
+use rstest::rstest;
+use serde_json::{Number, Value, json};
+
+use super::support::{LiveCall, assert_text, call, complete, request, route, stream_events};
+
+const PROVIDER: &str = "edenai";
+
+#[rstest]
+#[case::basic(json!({"messages": [{"role": "user", "content": "Say hello."}]}))]
+#[case::system_text(json!({
+    "system": "You are a helpful assistant.",
+    "messages": [{"role": "user", "content": "Say hello."}]
+}))]
+#[case::portable_cache_hint(json!({
+    "system": [{"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
+    "messages": [{"role": "user", "content": "Say hello."}]
+}))]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn completion_and_cost_handoff(route: MessagesRoute, #[case] payload: Value) {
+    within_deadline(async {
+        let host = LiveCall::new(PROVIDER);
+        let message = complete(request(&route, call(PROVIDER, payload), &host).await);
+        assert_text(&message);
+        host.assert_provider_result();
+        let expected_cost = message
+            .extra
+            .get("cost")
+            .and_then(|value| match value {
+                Value::Number(number) => number.as_f64(),
+                Value::String(text) => text.parse::<f64>().ok(),
+                _ => None,
+            })
+            .filter(|cost| *cost >= 0.0)
+            .and_then(Number::from_f64)
+            .map_or(Cost::Deferred, |amount| Cost::Reported { amount });
+        assert_eq!(host.facts().cost, expected_cost);
+        println!("{}", serde_json::to_string(&message).unwrap());
+    })
+    .await;
+}
+
+#[rstest]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn streaming(route: MessagesRoute) {
+    within_deadline(async {
+        let host = LiveCall::new(PROVIDER);
+        let events = stream_events(
+            request(
+                &route,
+                call(
+                    PROVIDER,
+                    json!({
+                        "stream": true, "messages": [{"role": "user", "content": "Say hello."}]
+                    }),
+                ),
+                &host,
+            )
+            .await,
+        )
+        .await;
+        assert!(events.iter().any(|event| {
+            event["delta"]["text"]
+                .as_str()
+                .is_some_and(|text| !text.is_empty())
+        }));
+        host.assert_provider_result();
+        assert_eq!(host.facts().cost, Cost::Deferred);
+    })
+    .await;
+}
+
+#[rstest]
+#[case::complete(false)]
+#[case::streaming(true)]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn tool_round_trip(route: MessagesRoute, #[case] stream: bool) {
+    super::support::tool_round_trip(
+        route,
+        PROVIDER,
+        stream,
+        json!({
+            "max_tokens": 512, "tool_choice": {"type": "auto"}
+        }),
+    )
+    .await;
+}

litellm-rust/crates/inference-messages/tests/live/support.rs

@@ -0,0 +1,265 @@
+use std::{sync::OnceLock, time::Duration};
+
+use futures_util::{FutureExt, TryStreamExt};
+use litellm_host::interceptors::{
+    ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest,
+};
+use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesRoute};
+use litellm_inference_testing::live::{LiveResources, model, within_deadline};
+use litellm_llms_types::formats::messages::MessagesResponse;
+use serde_json::{Value, json};
+
+pub(super) struct LiveCall {
+    provider: &'static str,
+    facts: OnceLock<ExecutionFacts>,
+}
+
+impl LiveCall {
+    pub(super) fn new(provider: &'static str) -> Self {
+        Self {
+            provider,
+            facts: OnceLock::new(),
+        }
+    }
+
+    pub(super) fn facts(&self) -> &ExecutionFacts {
+        self.facts
+            .get()
+            .expect("the route must deliver execution facts")
+    }
+
+    pub(super) fn assert_provider_result(&self) {
+        let facts = self.facts();
+        assert_eq!(facts.provider.provider, self.provider);
+        assert_eq!(facts.source, ResultSource::Provider);
+        println!(
+            "{}",
+            json!({"route": "messages", "provider": self.provider, "source": "provider"})
+        );
+    }
+}
+
+impl Interceptors<Error> for LiveCall {
+    async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
+        assert!(
+            self.facts.set(facts).is_ok(),
+            "result must be accepted exactly once"
+        );
+        Ok(())
+    }
+
+    async fn before_provider_request(
+        &self,
+        wire: WireRequest,
+        context: RequestContext,
+    ) -> Result<WireRequest, Error> {
+        assert_eq!(context.custom_llm_provider, self.provider);
+        println!(
+            "{}",
+            json!({"route": "messages", "provider": self.provider, "model": context.model})
+        );
+        Ok(wire)
+    }
+
+    async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+#[rstest::fixture]
+pub(super) fn route() -> MessagesRoute {
+    let resources = LiveResources::default();
+    MessagesRoute::new(resources.http, resources.auth, resources.secrets)
+}
+
+pub(super) async fn request(
+    route: &MessagesRoute,
+    call: MessagesCall,
+    host: &LiveCall,
+) -> MessagesCallResponse {
+    route
+        .execute(call, host, None)
+        .boxed()
+        .await
+        .expect("live request failed")
+}
+
+pub(super) fn call(provider: &str, payload: Value) -> MessagesCall {
+    let Value::Object(fields) = payload else {
+        panic!("live payload must be an object")
+    };
+    let body = serde_json::from_value(Value::Object(
+        [("max_tokens".into(), json!(128))]
+            .into_iter()
+            .chain(fields)
+            .chain([("model".into(), json!(model("messages", provider)))])
+            .collect(),
+    ))
+    .expect("live payload must be a valid Messages request");
+    MessagesCall {
+        body,
+        api_key: None,
+        api_base: None,
+        custom_llm_provider: Some(provider.into()),
+        litellm_params: Default::default(),
+        extra_headers: None,
+        provider_specific_header: None,
+        timeout: Some(Duration::from_secs(60)),
+        shaping: Default::default(),
+    }
+}
+
+pub(super) fn complete(response: MessagesCallResponse) -> Box<MessagesResponse> {
+    match response {
+        MessagesCallResponse::Complete(message) => message,
+        MessagesCallResponse::Stream { .. } => panic!("a complete request returned a stream"),
+    }
+}
+
+pub(super) fn assert_text(message: &MessagesResponse) {
+    assert!(!message.id.is_empty());
+    assert_eq!(message.role, "assistant");
+    assert!(message.stop_reason.is_some());
+    assert!(
+        message
+            .content
+            .iter()
+            .any(|block| block["text"].as_str().is_some_and(|text| !text.is_empty()))
+    );
+    assert!(
+        message
+            .usage
+            .as_ref()
+            .and_then(|usage| usage["output_tokens"].as_u64())
+            .is_some_and(|count| count > 0)
+    );
+}
+
+pub(super) async fn stream_events(response: MessagesCallResponse) -> Vec<Value> {
+    let MessagesCallResponse::Stream { head, chunks } = response else {
+        panic!("stream request returned a complete response")
+    };
+    let content_types: Vec<_> = head
+        .headers
+        .iter()
+        .filter(|(name, _)| name.eq_ignore_ascii_case("content-type"))
+        .map(|(_, value)| value.split(';').next().unwrap().trim())
+        .collect();
+    assert_eq!(content_types, ["text/event-stream"]);
+    let frames = litellm_framer::frames(chunks, litellm_framer::sse::SseCodec::default())
+        .try_collect::<Vec<_>>()
+        .await
+        .expect("live SSE framing failed");
+    let data_frames = match frames.last() {
+        Some(frame) if frame.data == "[DONE]" => &frames[..frames.len() - 1],
+        _ => frames.as_slice(),
+    };
+    let events: Vec<Value> = data_frames
+        .iter()
+        .map(|frame| serde_json::from_str(&frame.data).expect("SSE data must be JSON"))
+        .collect();
+    assert_eq!(
+        events.first().map(|event| event["type"].as_str()),
+        Some(Some("message_start"))
+    );
+    assert_eq!(
+        events.last().map(|event| event["type"].as_str()),
+        Some(Some("message_stop"))
+    );
+    assert!(!events.iter().any(|event| event["type"] == "error"));
+    assert!(events.iter().any(|event| {
+        event["usage"]["output_tokens"]
+            .as_u64()
+            .is_some_and(|count| count > 0)
+    }));
+    println!("{}", json!({"events": events}));
+    events
+}
+
+fn prompt() -> Value {
+    json!({"role": "user", "content": "Use the echo tool with value live-check, then tell me the result."})
+}
+
+fn tools() -> Value {
+    json!([{"name": "echo", "description": "Return a value", "input_schema": {
+        "type": "object", "properties": {"value": {"type": "string", "enum": ["live-check"]}}, "required": ["value"]
+    }}])
+}
+
+fn fragments(events: &[Value], index: u64, field: &str) -> String {
+    events
+        .iter()
+        .filter(|event| event["index"].as_u64() == Some(index))
+        .filter_map(|event| event["delta"][field].as_str())
+        .collect()
+}
+
+fn streamed_block(start: &Value, events: &[Value]) -> Value {
+    let index = start["index"]
+        .as_u64()
+        .expect("content block must have an index");
+    let block = &start["content_block"];
+    match block["type"].as_str() {
+        Some("tool_use") => {
+            let partial = fragments(events, index, "partial_json");
+            let input = match partial.is_empty() {
+                true => block["input"].clone(),
+                false => {
+                    serde_json::from_str(&partial).expect("streamed tool arguments must be JSON")
+                }
+            };
+            json!({"type": "tool_use", "id": block["id"], "name": block["name"], "input": input})
+        }
+        Some("text") => {
+            json!({"type": "text", "text": format!("{}{}", block["text"].as_str().unwrap_or_default(), fragments(events, index, "text"))})
+        }
+        Some("thinking") => {
+            json!({"type": "thinking", "thinking": format!("{}{}", block["thinking"].as_str().unwrap_or_default(), fragments(events, index, "thinking")),
+            "signature": format!("{}{}", block["signature"].as_str().unwrap_or_default(), fragments(events, index, "signature"))})
+        }
+        _ => block.clone(),
+    }
+}
+
+pub(super) async fn tool_round_trip(
+    route: MessagesRoute,
+    provider: &'static str,
+    stream: bool,
+    tool_params: Value,
+) {
+    within_deadline(async {
+        let first_host = LiveCall::new(provider);
+        let payload = json!({"messages": [prompt()], "tools": tools(), "stream": stream});
+        let fields = payload.as_object().expect("tool payload must be an object").clone();
+        let params = tool_params.as_object().expect("tool parameters must be an object").clone();
+        let first = request(&route, call(provider, Value::Object(fields.into_iter().chain(params).collect())), &first_host).await;
+        let content = match stream {
+            false => {
+                let message = complete(first);
+                assert_eq!(message.stop_reason.as_deref(), Some("tool_use"));
+                message.content
+            }
+            true => {
+                let events = stream_events(first).await;
+                assert!(events.iter().any(|event| event["delta"]["stop_reason"] == "tool_use"));
+                events.iter().filter(|event| event["type"] == "content_block_start")
+                    .map(|event| streamed_block(event, &events)).collect()
+            }
+        };
+        let tool = content.iter().find(|block| block["type"] == "tool_use").expect("requested tool must be returned");
+        assert_eq!(tool["name"], "echo");
+        assert_eq!(tool["input"]["value"], "live-check");
+        let tool_id = tool["id"].as_str().filter(|id| !id.is_empty()).expect("tool must have an id");
+        first_host.assert_provider_result();
+        let second_host = LiveCall::new(provider);
+        let second = complete(request(&route, call(provider, json!({
+            "tools": tools(), "messages": [prompt(),
+                {"role": "assistant", "content": content},
+                {"role": "user", "content": [{"type": "tool_result", "tool_use_id": tool_id, "content": "live-check"}]}]
+        })), &second_host).await);
+        assert_text(&second);
+        second_host.assert_provider_result();
+        println!("{}", serde_json::to_string(&second).unwrap());
+    })
+    .await;
+}

litellm-rust/crates/inference-messages/tests/live/vertex_ai.rs

@@ -0,0 +1,68 @@
+use litellm_inference_messages::MessagesRoute;
+use litellm_inference_testing::live::within_deadline;
+use rstest::rstest;
+use serde_json::{Value, json};
+
+use super::support::{LiveCall, assert_text, call, complete, request, route, stream_events};
+
+const PROVIDER: &str = "vertex_ai";
+
+#[rstest]
+#[case::basic(json!({"messages": [{"role": "user", "content": "Say hello."}]}))]
+#[case::system_text(json!({
+    "system": "You are a helpful assistant.",
+    "messages": [{"role": "user", "content": "Say hello."}]
+}))]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn completion(route: MessagesRoute, #[case] payload: Value) {
+    within_deadline(async {
+        let host = LiveCall::new(PROVIDER);
+        let message = complete(request(&route, call(PROVIDER, payload), &host).await);
+        assert_text(&message);
+        host.assert_provider_result();
+        println!("{}", serde_json::to_string(&message).unwrap());
+    })
+    .await;
+}
+
+#[rstest]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn streaming(route: MessagesRoute) {
+    within_deadline(async {
+        let host = LiveCall::new(PROVIDER);
+        let response = request(
+            &route,
+            call(
+                PROVIDER,
+                json!({"stream": true, "messages": [{"role": "user", "content": "Say hello."}]}),
+            ),
+            &host,
+        )
+        .await;
+        let events = stream_events(response).await;
+        assert!(events.iter().any(|event| {
+            event["delta"]["text"]
+                .as_str()
+                .is_some_and(|text| !text.is_empty())
+        }));
+        host.assert_provider_result();
+    })
+    .await;
+}
+
+#[rstest]
+#[case::complete(false)]
+#[case::streaming(true)]
+#[ignore = "calls a real provider, requires credentials and a live model"]
+#[tokio::test]
+async fn tool_round_trip(route: MessagesRoute, #[case] stream: bool) {
+    super::support::tool_round_trip(
+        route,
+        PROVIDER,
+        stream,
+        json!({"tool_choice": {"type": "auto"}}),
+    )
+    .await;
+}

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -894,3 +894,38 @@ async fn provider_validation_runs_before_caller_parameter_removal(
     );
     assert!(received(&upstream).await.is_empty());
 }
+
+#[rstest]
+#[tokio::test]
+async fn edenai_preserves_native_thinking_and_strips_billing_and_cache_extensions(
+    call: MessagesCall,
+) {
+    let upstream = upstream([message_response()]).await;
+    let thinking = json!({"type": "enabled", "budget_tokens": 512, "native_field": true});
+    run_message(MessagesCall {
+        custom_llm_provider: Some("edenai".into()),
+        api_base: Some(format!("{}/v3", upstream.uri())),
+        api_key: Some("eden-key".into()),
+        extra_headers: headers([("Anthropic-Beta", "unregistered-native-beta")]),
+        ..with_fields(call, json!({
+            "thinking": thinking, "temperature": 0.25,
+            "system": [{"type": "text", "text": "x-anthropic-billing-header: hidden"},
+                       {"type": "text", "text": "keep", "cache_control": {"type": "ephemeral", "ttl": "1h", "scope": "global"}}],
+        }))
+    }).await;
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v3/v1/messages");
+    assert_eq!(sent.header("authorization"), Some("Bearer eden-key"));
+    assert_eq!(
+        sent.header("anthropic-beta"),
+        Some("unregistered-native-beta")
+    );
+    assert_eq!(
+        sent.json(),
+        json!({
+            "model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}],
+            "thinking": thinking, "temperature": 0.25,
+            "system": [{"type": "text", "text": "keep", "cache_control": {"type": "ephemeral"}}],
+        })
+    );
+}

litellm-rust/crates/inference-messages/tests/messages/stream.rs

@@ -348,6 +348,7 @@ async fn the_timeout_covers_a_stalled_stream_body(call: MessagesCall) {
 #[rstest]
 #[case::anthropic("anthropic")]
 #[case::azure_ai("azure_ai")]
+#[case::edenai("edenai")]
 #[tokio::test]
 async fn the_sdk_returns_stream_headers_and_every_sse_byte(
     call: MessagesCall,
@@ -358,6 +359,7 @@ async fn the_sdk_returns_stream_headers_and_every_sse_byte(
         .execute(
             MessagesCall {
                 custom_llm_provider: Some(provider.into()),
+                api_key: Some("stream-key".into()),
                 litellm_params: Default::default(),
                 ..streaming(call, upstream.uri())
             },

litellm-rust/crates/inference-testing/Cargo.toml

@@ -7,7 +7,9 @@ repository.workspace = true
 publish = false
 
 [dependencies]
+litellm-auth.workspace = true
 futures-util.workspace = true
 litellm-http.workspace = true
 litellm-inference.workspace = true
 litellm-secrets.workspace = true
+tokio.workspace = true

litellm-rust/crates/inference-testing/README.md

@@ -0,0 +1,46 @@
+# Live provider route tests
+
+Each route crate owns `tests/live.rs`. Use named `rstest` provider cases for the behaviors that route supports. Shared setup lives in `litellm_inference_testing::live`, with production HTTP settings, authentication services, environment secrets and a deadline
+
+These tests execute the Rust route against a real provider. They do not prove Python parity, proxy authorization, billing persistence or agent compatibility. Those belong in the corresponding boundary and gateway suites
+
+All live cases use `#[ignore = "calls a real provider, requires credentials and a live model"]`. Normal Cargo runs report them as ignored. An explicitly selected case fails on missing credentials, unavailable models, provider errors or timeouts. Never return early and count missing coverage as a pass
+
+Set `LITELLM_LIVE_<ROUTE>_<PROVIDER>_MODEL` to the latest model available to the test account. The model has no compiled default because vendor availability changes independently of LiteLLM. Credentials use the provider's existing configuration rather than a second test-only key convention. Export credentials in the shell before running tests. The harness does not load `.env` or modify process environment while tests run
+
+## Messages
+
+Completion checks text and usage. Streaming consumes SSE to completion and checks text, usage and the terminal event. Tool round trips request a tool call and send its returned ID back with a result. Each case checks provider execution facts and rejects cache results
+
+Messages setup and assertions live in `inference-messages/tests/live/support.rs`. Provider modules under `tests/live/` select their payloads and tool policy. The shared helpers consume streams completely and reconstruct text, thinking signatures and tool arguments for the follow-up request
+
+The initial provider cases are `vertex_ai` and `edenai`. A registered ignored case is available coverage, not evidence that it passed against that provider
+
+Run one provider's cases explicitly
+
+```bash
+LITELLM_LIVE_MESSAGES_VERTEX_AI_MODEL=<claude-model-on-vertex> \
+  cargo test --manifest-path litellm-rust/Cargo.toml \
+  -p litellm-inference-messages --test live vertex_ai \
+  -- --ignored --nocapture --test-threads=1
+```
+
+For Vertex AI, set `LITELLM_LIVE_MESSAGES_VERTEX_AI_MODEL` to a Claude model published in the project's location. Credentials, project and location resolve through the production Rust GCP authentication, from `GOOGLE_APPLICATION_CREDENTIALS` or `VERTEXAI_CREDENTIALS`, with `VERTEXAI_PROJECT` and `VERTEXAI_LOCATION`
+
+Vertex AI cases live in `inference-messages/tests/live/vertex_ai.rs`. They cover plain and system messages, text streaming, and complete and streaming tool round trips. Tool cases use automatic selection because current Claude models reject forced `tool_choice`. Each case still requires a tool call, checks its arguments, and sends the tool result back
+
+For EdenAI, set `LITELLM_LIVE_MESSAGES_EDENAI_MODEL` and `EDENAI_API_KEY`, and select `edenai`. EdenAI requires a production token for live coverage, because sandbox tokens return dummy responses
+
+EdenAI cases live in `inference-messages/tests/live/edenai.rs`. They cover system messages, portable cache hints, provider-reported cost handoff, text streaming, and complete and streaming tool round trips. The cost assertion compares the public response with the execution facts instead of pinning a vendor price
+EdenAI tool cases use automatic selection because current Claude models reject forced `tool_choice`. The test still requires a tool call and verifies its arguments and follow-up response
+
+Other tool cases require a model that supports forced `tool_choice`, except where a provider's section says otherwise. A provider rejection fails the selected case instead of skipping it
+
+Select one behavior by its test name, for example `completion::case_1_basic`. To list every available cell without contacting any provider
+
+```bash
+cargo test --manifest-path litellm-rust/Cargo.toml \
+  -p litellm-inference-messages --test live -- --list
+```
+
+For another route, reuse `LiveResources`, `model` and `within_deadline`. Keep payload construction and response assertions in that route's test file. Add only supported provider behaviors, and document any gap instead of substituting mocks or skipping failures. Run cases serially to keep spending and rate limits predictable

litellm-rust/crates/inference-testing/src/lib.rs

@@ -9,6 +9,8 @@ use litellm_secrets::{SecretValue, source::SecretSource};
 
 use litellm_inference::resources::CoreResources;
 
+pub mod live;
+
 pub fn http_pool() -> HttpClientPool {
     HttpClientPool::new(Arc::new(PublicDnsResolver))
 }

litellm-rust/crates/inference-testing/src/live.rs

@@ -0,0 +1,53 @@
+use std::{future::Future, sync::Arc, time::Duration};
+
+use litellm_http::{ClientVariant, HttpSettings, HttpSettingsLayer, Resolution};
+use litellm_inference::resources::CoreResources;
+use litellm_secrets::source::{EnvironmentSecrets, SecretSource};
+
+pub const DEADLINE: Duration = Duration::from_secs(120);
+
+pub struct LiveResources {
+    pub http: litellm_http::Client,
+    pub auth: Arc<litellm_auth::AuthServices>,
+    pub secrets: Arc<dyn SecretSource>,
+}
+
+impl Default for LiveResources {
+    fn default() -> Self {
+        let resources = CoreResources::new(Arc::new(super::http_pool()));
+        let settings =
+            HttpSettings::from_layers([HttpSettingsLayer::from_environment(&|key: &str| {
+                std::env::var(key).ok()
+            })]);
+        let resolution = Resolution::from(&settings);
+        let http = resources
+            .pool
+            .client(&resolution.config, ClientVariant::Provider)
+            .expect("live provider HTTP settings must be valid");
+        let secrets_http = resources
+            .pool
+            .client(&resolution.config, ClientVariant::NoRedirect)
+            .expect("live secret HTTP settings must be valid");
+        Self {
+            http,
+            auth: resources.auth,
+            secrets: Arc::new(EnvironmentSecrets::python_compatible(secrets_http)),
+        }
+    }
+}
+
+pub fn model(route: &str, provider: &str) -> String {
+    let name = format!("LITELLM_LIVE_{route}_{provider}_MODEL").to_ascii_uppercase();
+    std::env::var(&name)
+        .ok()
+        .filter(|value| !value.trim().is_empty())
+        .unwrap_or_else(|| {
+            panic!("set {name} to an available current model before running this live case")
+        })
+}
+
+pub async fn within_deadline<T>(operation: impl Future<Output = T>) -> T {
+    tokio::time::timeout(DEADLINE, operation)
+        .await
+        .expect("live case exceeded its 120 second deadline")
+}

litellm-rust/crates/inference/src/caching.rs

@@ -13,7 +13,9 @@ use litellm_cache_response::{
 };
 use litellm_host::{
     call::{CallOutput, OutputOf},
-    interceptors::{ExecutionFacts, Interceptors, ProviderIdentity, ResultSource, WireRequest},
+    interceptors::{
+        Cost, ExecutionFacts, Interceptors, ProviderIdentity, ResultSource, WireRequest,
+    },
     lifecycle::{CallEvent, ExecutionEvent},
     observation::ObservationSender,
     protocol::Protocol,
@@ -168,10 +170,12 @@ where
         None => (provider().await?, ResultSource::Provider),
     };
     let from_provider = source == ResultSource::Provider;
+    let cost = Cost::settle(&source, None);
     publish(
         ExecutionFacts {
             provider: identity,
             source,
+            cost,
         },
         interceptors,
         observers,
@@ -212,6 +216,7 @@ where
     publish(
         ExecutionFacts {
             provider: identity,
+            cost: Cost::settle(&source, None),
             source: source.clone(),
         },
         interceptors,

litellm-rust/crates/llms/AGENTS.md

@@ -24,7 +24,7 @@
 
 - Every file is `# rules` then `# references`, both concise bullets. Split a long `# rules` into `##` topic sections
 - A folder gets an AGENTS.md only when it adds something: a deviation, an explicit reuse of another provider, or upstream docs no other file lists. A folder without one follows its nearest parent
-- Every `src/<provider>/<format>/` folder gets one, since it talks to its own upstream endpoint. `.greptile/files.json` lists them all
+- Every `src/<provider>/<format>/` folder gets one, since it talks to its own upstream endpoint
 - A rule lives in the highest file where it holds. Never restate a parent's rule
 - Each upstream URL appears in exactly one AGENTS.md, the one owning that contract. Others point to that file by path
   - Format specs: `litellm-llms-types/src/formats/<format>/`

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -41,6 +41,10 @@ pub trait BaseMessagesConfig: Sync {
         Ok(response)
     }
 
+    fn reported_cost(&self, _response: &MessagesResponse) -> Option<serde_json::Number> {
+        None
+    }
+
     fn secret_names(&self) -> &'static [&'static str];
 
     /// Shapes the forwarded headers and names the credential, the way Python's

litellm-rust/crates/llms/src/edenai/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/edenai/messages/transformation.rs

@@ -0,0 +1,120 @@
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::settings::resolve_non_empty;
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
+use litellm_llms_types::providers::anthropic::DEFAULT_HEADERS;
+use litellm_router_types::LitellmParams;
+use serde_json::Value;
+
+use crate::{
+    Error,
+    anthropic::{
+        common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
+        messages::transformation::update_headers_with_anthropic_beta,
+    },
+    base_llm::{
+        auth::AuthScheme,
+        messages::{
+            context::MessagesTransformContext,
+            transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment},
+        },
+    },
+    openai_like::messages::transformation::{complete_messages_url, portable_cache_control},
+};
+
+const API_KEY_ENV: &str = "EDENAI_API_KEY";
+const API_BASE_ENV: &str = "EDENAI_API_BASE";
+const DEFAULT_BASE: &str = "https://api.edenai.run/v3";
+
+pub struct EdenAIAnthropicMessagesConfig;
+pub const EDENAI_MESSAGES_CONFIG: EdenAIAnthropicMessagesConfig = EdenAIAnthropicMessagesConfig;
+
+impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
+            litellm_auth::Error::MissingApiKey {
+                provider: "Eden AI",
+                environment_variable: API_KEY_ENV,
+            },
+        ))?;
+        if has_anthropic_credential(&headers) {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        Ok(ValidatedEnvironment {
+            headers,
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Bearer,
+                secret: SecretValue::new(key),
+            },
+        })
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _params: &LitellmParams,
+        _stream: bool,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let base = resolve_non_empty(api_base, env, &[API_BASE_ENV])
+            .unwrap_or_else(|| DEFAULT_BASE.into());
+        Ok(complete_messages_url(&base))
+    }
+
+    fn transform_anthropic_messages_request(
+        &self,
+        request: MessagesRequest,
+        _context: &MessagesTransformContext,
+    ) -> Result<MessagesRequest, Error> {
+        Ok(MessagesRequest {
+            params: MessagesOptionalParams {
+                system: request
+                    .params
+                    .system
+                    .and_then(filter_billing_headers_from_system),
+                ..request.params
+            },
+            ..request
+        })
+    }
+
+    fn secret_names(&self) -> &'static [&'static str] {
+        &[API_KEY_ENV, API_BASE_ENV]
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        DEFAULT_HEADERS
+    }
+
+    fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
+        update_headers_with_anthropic_beta(headers, request)
+    }
+
+    fn wire_body(&self, body: Value) -> Value {
+        portable_cache_control(body)
+    }
+
+    fn reported_cost(&self, response: &MessagesResponse) -> Option<serde_json::Number> {
+        let cost = match response.extra.get("cost")? {
+            Value::Number(cost) => cost.as_f64()?,
+            Value::String(cost) => cost.parse::<f64>().ok()?,
+            _ => return None,
+        };
+        if cost < 0.0 {
+            return None;
+        }
+        serde_json::Number::from_f64(cost)
+    }
+}

litellm-rust/crates/llms/src/edenai/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/llms/src/lib.rs

@@ -6,6 +6,7 @@ pub mod bedrock;
 pub mod bedrock_mantle;
 pub mod cohere;
 pub mod deepseek;
+pub mod edenai;
 mod error;
 pub mod minimax;
 pub mod mistral;

litellm-rust/crates/llms/src/openai_like/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/openai_like/messages/transformation.rs

@@ -0,0 +1,100 @@
+use serde_json::{Map, Value};
+
+pub fn complete_messages_url(base: &str) -> String {
+    let base = base.trim_end_matches('/');
+    if base.ends_with("/v1/messages") {
+        return base.into();
+    }
+    format!("{}/v1/messages", base.strip_suffix("/v1").unwrap_or(base))
+}
+
+pub fn portable_cache_control(body: Value) -> Value {
+    let Value::Object(fields) = body else {
+        return body;
+    };
+    Value::Object(
+        portable_block(fields)
+            .into_iter()
+            .map(|(key, value)| {
+                let value = match key.as_str() {
+                    "system" | "tools" => map_blocks(value, portable_value),
+                    "messages" => map_blocks(value, portable_message),
+                    _ => value,
+                };
+                (key, value)
+            })
+            .collect(),
+    )
+}
+
+fn portable_block(fields: Map<String, Value>) -> Map<String, Value> {
+    fields
+        .into_iter()
+        .filter_map(|(key, value)| {
+            if key != "cache_control" {
+                return Some((key, value));
+            }
+            let Value::Object(cache) = value else {
+                return None;
+            };
+            let kind = cache
+                .get("type")
+                .and_then(Value::as_str)
+                .unwrap_or("ephemeral");
+            Some((key, serde_json::json!({"type": kind})))
+        })
+        .collect()
+}
+
+fn portable_value(value: Value) -> Value {
+    match value {
+        Value::Object(fields) => Value::Object(portable_block(fields)),
+        other => other,
+    }
+}
+
+fn map_blocks(value: Value, map: fn(Value) -> Value) -> Value {
+    match value {
+        Value::Array(blocks) => Value::Array(blocks.into_iter().map(map).collect()),
+        other => other,
+    }
+}
+
+fn portable_message(value: Value) -> Value {
+    let Value::Object(fields) = value else {
+        return value;
+    };
+    Value::Object(
+        fields
+            .into_iter()
+            .map(|(key, value)| {
+                let value = if key == "content" {
+                    map_blocks(value, portable_content)
+                } else {
+                    value
+                };
+                (key, value)
+            })
+            .collect(),
+    )
+}
+
+fn portable_content(value: Value) -> Value {
+    let Value::Object(fields) = value else {
+        return value;
+    };
+    let tool_result = fields.get("type").and_then(Value::as_str) == Some("tool_result");
+    Value::Object(
+        portable_block(fields)
+            .into_iter()
+            .map(|(key, value)| {
+                let value = if tool_result && key == "content" {
+                    map_blocks(value, portable_value)
+                } else {
+                    value
+                };
+                (key, value)
+            })
+            .collect(),
+    )
+}

litellm-rust/crates/llms/src/openai_like/mod.rs

@@ -1,2 +1,3 @@
 pub mod chat;
 pub mod common_utils;
+pub mod messages;

litellm-rust/crates/llms/tests/edenai_messages_transformation.rs

@@ -0,0 +1,34 @@
+use litellm_llms::{
+    base_llm::messages::transformation::BaseMessagesConfig,
+    edenai::messages::transformation::EDENAI_MESSAGES_CONFIG,
+};
+use litellm_llms_types::formats::messages::MessagesResponse;
+use rstest::rstest;
+use serde_json::{Value, json};
+
+#[rstest]
+#[case::number(json!(0.025), Some(0.025))]
+#[case::free(json!(0), Some(0.0))]
+#[case::text(json!("0.75"), Some(0.75))]
+#[case::absent(Value::Null, None)]
+#[case::malformed(json!("not-a-cost"), None)]
+#[case::negative(json!(-1), None)]
+#[case::boolean(json!(true), None)]
+#[case::infinite(json!("inf"), None)]
+#[case::nan(json!("NaN"), None)]
+fn reported_cost_is_a_finite_nonnegative_amount(
+    #[case] cost: Value,
+    #[case] expected: Option<f64>,
+) {
+    let response: MessagesResponse = serde_json::from_value(json!({
+        "id": "msg_test", "type": "message", "role": "assistant", "model": "native-test",
+        "content": [], "stop_reason": "end_turn", "stop_sequence": null, "cost": cost,
+    }))
+    .unwrap();
+    assert_eq!(
+        EDENAI_MESSAGES_CONFIG
+            .reported_cost(&response)
+            .and_then(|cost| cost.as_f64()),
+        expected
+    );
+}

litellm-rust/crates/llms/tests/openai_like_messages_transformation.rs

@@ -0,0 +1,64 @@
+use litellm_llms::openai_like::messages::transformation::{
+    complete_messages_url, portable_cache_control,
+};
+use rstest::rstest;
+use serde_json::{Value, json};
+
+#[rstest]
+#[case::root("https://native.test", "https://native.test/v1/messages")]
+#[case::trailing_slash("https://native.test/", "https://native.test/v1/messages")]
+#[case::versioned("https://native.test/v1/", "https://native.test/v1/messages")]
+#[case::full_url(
+    "https://native.test/v3/v1/messages/",
+    "https://native.test/v3/v1/messages"
+)]
+#[case::gateway_prefix("https://native.test/v3", "https://native.test/v3/v1/messages")]
+fn messages_url_appends_the_endpoint_once(#[case] base: &str, #[case] expected: &str) {
+    assert_eq!(complete_messages_url(base), expected);
+}
+
+#[rstest]
+#[case::extended(json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), Some(json!({"type": "ephemeral"})))]
+#[case::missing_type(json!({"ttl": "1h"}), Some(json!({"type": "ephemeral"})))]
+#[case::invalid_type(json!({"type": 7}), Some(json!({"type": "ephemeral"})))]
+#[case::null(Value::Null, None)]
+#[case::scalar(json!(false), None)]
+fn portable_cache_only_rewrites_protocol_cache_hints(
+    #[case] cache: Value,
+    #[case] expected_cache: Option<Value>,
+) {
+    let application_data = json!({"cache_control": cache, "nested": {"cache_control": cache}});
+    let body = json!({
+        "cache_control": cache,
+        "model": "native-test",
+        "system": [{"type": "text", "text": "system", "cache_control": cache}],
+        "tools": [{"name": "tool", "cache_control": cache, "input_schema": application_data}],
+        "messages": [{"role": "user", "content": [
+            {"type": "tool_use", "input": application_data, "cache_control": cache},
+            {"type": "tool_result", "content": [{"type": "text", "text": "result", "cache_control": cache}], "cache_control": cache},
+        ]}],
+    });
+    let hint = |value: Value| {
+        let fields = value.as_object().unwrap().clone();
+        Value::Object(
+            fields
+                .into_iter()
+                .chain(
+                    expected_cache
+                        .clone()
+                        .map(|cache| ("cache_control".into(), cache)),
+                )
+                .collect(),
+        )
+    };
+    let expected = hint(json!({
+        "model": "native-test",
+        "system": [hint(json!({"type": "text", "text": "system"}))],
+        "tools": [hint(json!({"name": "tool", "input_schema": application_data}))],
+        "messages": [{"role": "user", "content": [
+            hint(json!({"type": "tool_use", "input": application_data})),
+            hint(json!({"type": "tool_result", "content": [hint(json!({"type": "text", "text": "result"}))]})),
+        ]}],
+    }));
+    assert_eq!(portable_cache_control(body), expected);
+}

litellm/__init__.py

@@ -477,6 +477,7 @@ max_end_user_budget_id: Optional[str] = None
 # pass through unchanged.
 validate_end_user_id_in_db: bool = False
 block_requests_for_models_without_pricing: bool = False
+personal_key_model_access_from_teams: bool = False
 disable_end_user_cost_tracking: Optional[bool] = None
 disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
 enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
@@ -721,6 +722,7 @@ aws_polly_models: Set = set()
 transcribe_models: Set = set()
 gigachat_models: Set = set()
 llamagate_models: Set = set()
+coralbricks_models: Set[str] = set()
 reducto_models: Set = set()
 bedrock_mantle_models: Set = set()
 
@@ -1012,6 +1014,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             gigachat_models.add(key)
         elif value.get("litellm_provider") == "llamagate":
             llamagate_models.add(key)
+        elif value.get("litellm_provider") == "coralbricks":
+            coralbricks_models.add(key)
         elif value.get("litellm_provider") == "reducto":
             reducto_models.add(key)
         elif value.get("litellm_provider") == "bedrock_mantle":
@@ -1146,6 +1150,7 @@ model_list = list(
     | docker_model_runner_models
     | reducto_models
     | bedrock_mantle_models
+    | coralbricks_models
     | set(clarifai_models)
 )
 
@@ -1265,6 +1270,7 @@ def _build_models_by_provider() -> dict:
         "transcribe": transcribe_models,
         "gigachat": gigachat_models,
         "llamagate": llamagate_models,
+        "coralbricks": coralbricks_models,
         "reducto": reducto_models,
         "bedrock_mantle": bedrock_mantle_models,
     }
@@ -1689,6 +1695,9 @@ if TYPE_CHECKING:
     from .llms.openrouter.decisions.transformation import (
         OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
     )
+    from .llms.fireworks_ai.decisions.transformation import (
+        FireworksAIDecisionsConfig as FireworksAIDecisionsConfig,
+    )
     from .llms.cloudflare.decisions.transformation import (
         CloudflareDecisionsConfig as CloudflareDecisionsConfig,
     )
@@ -2277,6 +2286,7 @@ if TYPE_CHECKING:
     supports_parallel_function_calling: Callable[..., bool]
     supports_vision: Callable[..., bool]
     supports_audio_input: Callable[..., bool]
+    supports_video_input: Callable[..., bool]
     supports_audio_output: Callable[..., bool]
     supports_system_messages: Callable[..., bool]
     supports_reasoning: Callable[..., bool]

litellm/_lazy_imports_registry.py

@@ -36,6 +36,7 @@ UTILS_NAMES: Final = (
     "supports_parallel_function_calling",
     "supports_vision",
     "supports_audio_input",
+    "supports_video_input",
     "supports_audio_output",
     "supports_system_messages",
     "supports_reasoning",
@@ -160,6 +161,7 @@ LLM_CONFIG_NAMES: Final = (
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
     "OpenRouterDecisionsConfig",
+    "FireworksAIDecisionsConfig",
     "CloudflareDecisionsConfig",
     "StrandsDeciderDecisionsConfig",
     "DatabricksDecisionsConfig",
@@ -481,6 +483,7 @@ _UTILS_IMPORT_MAP: Final = {
     ),
     "supports_vision": (".utils", "supports_vision"),
     "supports_audio_input": (".utils", "supports_audio_input"),
+    "supports_video_input": (".utils", "supports_video_input"),
     "supports_audio_output": (".utils", "supports_audio_output"),
     "supports_system_messages": (".utils", "supports_system_messages"),
     "supports_reasoning": (".utils", "supports_reasoning"),
@@ -722,6 +725,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
+    "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
     "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
     "StrandsDeciderDecisionsConfig": (
         ".llms.strands_decider.decisions.transformation",

litellm/caching/caching.py

@@ -445,7 +445,7 @@ class Cache:
 
         1. If caching groups are set, then return the caching group as the model https://docs.litellm.ai/docs/routing#caching-across-model-groups
         2. Else if a model_group is set, then return the model_group as the model. This is used for all requests sent through the litellm.Router()
-        3. Else use the `model` passed in kwargs
+        3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component)
         """
         litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
         metadata_sources: Final[tuple[dict, ...]] = (
@@ -460,7 +460,10 @@ class Cache:
         caching_group: Final = next(
             (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None
         )
-        return caching_group or model_group or kwargs["model"]
+        model_id_suffix: Final = (
+            f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else ""
+        )
+        return caching_group or model_group or f"{kwargs['model']}{model_id_suffix}"
 
     def _get_caching_group(self, metadata: dict, model_group: str | None) -> str | None:
         caching_groups: Final[list | None] = metadata.get("caching_groups", [])

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -74,7 +74,7 @@ if TYPE_CHECKING:
         ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
-    from litellm.types.utils import Choices
+    from litellm.types.utils import Choices, Message
 
 
 _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@@ -312,6 +312,67 @@ def _as_chat_reasoning_items(
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
+    shifted: Final = {
+        key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
+        for key, value in annotation.items()
+    }
+    return cast(ChatCompletionAnnotation, shifted)
+
+
+def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
+    annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
+    return annotations or ()
+
+
+def _message_reasoning_content(message: "Message") -> str | None:
+    reasoning_content: str | None = getattr(message, "reasoning_content", None)
+    return reasoning_content
+
+
+def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
+    reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
+    return reasoning_items or ()
+
+
+def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
+    for message, offset in zip(messages, offsets):
+        for annotation in _message_annotations(message):
+            yield _shift_annotation(annotation, offset)
+
+
+def _fold_choices_into_one(
+    choices: Sequence["Choices"],
+    tool_calls: Sequence[Mapping[str, object]],
+    trailing_reasoning_content: str | None,
+    trailing_reasoning_item: _BuiltReasoningItem | None,
+) -> "Choices":
+    """Clients that only read choices[0] must still see every text and tool call."""
+    from litellm.types.utils import Choices, Message
+
+    messages: Final = tuple(choice.message for choice in choices)
+    texts: Final = tuple(str(message.content or "") for message in messages)
+    offsets: Final = (0, *accumulate(len(text) for text in texts))
+    annotations: Final = tuple(_shifted_annotations(messages, offsets))
+    reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
+    reasoning_items: Final = (
+        *chain.from_iterable(_message_reasoning_items(m) for m in messages),
+        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+    )
+    return Choices(
+        message=Message(
+            role=messages[0].role if messages else "assistant",
+            content="".join(texts) if messages else None,
+            annotations=list(annotations) or None,
+            tool_calls=list(tool_calls) or None,
+            reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        ),
+        finish_reason="tool_calls" if tool_calls else "stop",
+        index=0,
+    )
+
+
 _ToolChoiceT = TypeVar("_ToolChoiceT")
 
 
@@ -958,47 +1019,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else:
                 pass  # don't fail request if item in list is not supported
 
-        if accumulated_tool_calls and choices:
-            last_choice: Final = choices[-1]
-            last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
-            last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
-            merged_reasoning_content: Final = (
-                " ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
-            )
-            merged_reasoning_items: Final = _as_chat_reasoning_items(
-                (
-                    *(last_reasoning_items or ()),
-                    *(() if pending_reasoning_item is None else (pending_reasoning_item,)),
-                )
-            )
-            merged_message: Final = Message(
-                role=last_choice.message.role,
-                content=last_choice.message.content,
-                annotations=getattr(last_choice.message, "annotations", None),
+        if not choices and not accumulated_tool_calls:
+            return choices
+        return [
+            _fold_choices_into_one(
+                choices=choices,
                 tool_calls=accumulated_tool_calls,
-                reasoning_content=merged_reasoning_content,
-                reasoning_items=merged_reasoning_items,
+                trailing_reasoning_content=reasoning_content,
+                trailing_reasoning_item=pending_reasoning_item,
             )
-            return [
-                *choices[:-1],
-                Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
-            ]
-
-        if accumulated_tool_calls:
-            msg = Message(
-                content=None,
-                tool_calls=accumulated_tool_calls,
-                reasoning_content=reasoning_content,
-                reasoning_items=cast(
-                    list[ChatCompletionReasoningItem] | None,
-                    ([pending_reasoning_item] if pending_reasoning_item is not None else None),
-                ),
-            )
-            choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
-            reasoning_content = None
-            pending_reasoning_item = None
-
-        return choices
+        ]
 
     @staticmethod
     def _build_empty_incomplete_choice(

litellm/constants.py

@@ -1007,6 +1007,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.meta.ai/v1",
     "https://api.sailresearch.com/v1",
     "https://api.cognition.ai/v1",
+    "https://inference.coralbricks.ai/v1",
     "https://api.cortecs.ai/v1",
     "https://api.scx.ai/v1",
     "https://api.prisminference.com/v1",
@@ -1083,6 +1084,7 @@ openai_compatible_providers: Final[list] = [
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
+    "coralbricks",
     "cortecs",
     "scx-ai",
     "prism",
@@ -1961,6 +1963,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "openai_system_messages_first",
     "max_ui_session_budget",
     "budget_rollover",
+    "personal_key_model_access_from_teams",
     "mcp_tool_search",
     "turn_off_message_logging",
     "datadog_params",

litellm/cost_calculator.py

@@ -18,6 +18,8 @@ from litellm.constants import (
     DEFAULT_MAX_LRU_CACHE_SIZE,
     DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
 )
+from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
+from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
@@ -570,7 +572,22 @@ def cost_per_token(
     ):  # Option 3. if user passed model="bedrock/anthropic.claude-3", use model="anthropic.claude-3"
         model = model_without_prefix
 
-    # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
+    if is_decisions_call_type(call_type):
+        decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+        decisions_overrides: Final = decisions_pricing_fields(
+            litellm.model_cost.get(decisions_model_info["key"], {})  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # model cost map is dynamically loaded
+        )
+        if decisions_overrides:
+            return generic_cost_per_token(
+                model=model,
+                usage=usage_block,
+                custom_llm_provider=custom_llm_provider,
+                service_tier=service_tier,
+                data_residency=data_residency,
+                model_info=cast(  # cast-ok: _decisions fields override same-named ModelInfo rates
+                    ModelInfo, {**decisions_model_info, **decisions_overrides}
+                ),
+            )
     if call_type == "speech" or call_type == "aspeech":
         lyria_generation_cost: Final = (
             get_vertex_ai_lyria_generation_cost(model=model_without_prefix)
@@ -1124,6 +1141,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj
         return "text_completion"
     elif isinstance(completion_response, LiteLLMSendMessageResponse):
         return "send_message"
+    elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
+        return "adecisions"
 
     return call_type
 
@@ -1134,6 +1153,8 @@ _infer_call_type = infer_call_type
 def _apply_cost_discount(
     base_cost: float,
     custom_llm_provider: str | None,
+    model: str | None = None,
+    region_name: str | None = None,
 ) -> tuple[float, float, float]:
     """
     Apply provider-specific cost discount from module-level config.
@@ -1141,17 +1162,20 @@ def _apply_cost_discount(
     Args:
         base_cost: The base cost before discount
         custom_llm_provider: The LLM provider name
+        model: The model name, used to resolve <provider>/<model-pattern> discount keys
+        region_name: The region segment the cost key may carry, so patterns can match the region-stripped name
 
     Returns:
         Tuple of (final_cost, discount_percent, discount_amount)
     """
     original_cost: Final = base_cost
-    discount_percent = 0.0
-    discount_amount = 0.0
+    resolved_discount: Final = resolve_cost_discount(
+        litellm.cost_discount_config, custom_llm_provider, model, region_name
+    )
+    discount_percent: Final = 0.0 if resolved_discount is None else resolved_discount
+    discount_amount: Final = original_cost * discount_percent
 
-    if custom_llm_provider and custom_llm_provider in litellm.cost_discount_config:
-        discount_percent = litellm.cost_discount_config[custom_llm_provider]
-        discount_amount = original_cost * discount_percent
+    if resolved_discount is not None:
         final_cost: Final = original_cost - discount_amount
 
         if verbose_logger.isEnabledFor(logging.DEBUG):
@@ -1472,6 +1496,7 @@ def completion_cost(
             )
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
+        custom_priced_request_model: Final = model if custom_pricing is True else None
         selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
@@ -1726,6 +1751,8 @@ def completion_cost(
                     ) = _apply_cost_discount(
                         base_cost=_final_cost,
                         custom_llm_provider=custom_llm_provider,
+                        model=model,
+                        region_name=region_name if isinstance(region_name, str) else None,
                     )
 
                     # Apply margin from module-level config if configured
@@ -1785,6 +1812,7 @@ def completion_cost(
                     litellm_logging_obj=litellm_logging_obj,
                     total_time=total_time,
                 )
+                model_for_discount = custom_priced_request_model or model
                 # Calculate cost based on prompt_tokens, completion_tokens
                 if (
                     "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
@@ -1893,6 +1921,8 @@ def completion_cost(
                     ) = _apply_cost_discount(
                         base_cost=_final_cost,
                         custom_llm_provider=custom_llm_provider,
+                        model=model_for_discount,
+                        region_name=region_name if isinstance(region_name, str) else None,
                     )
                 else:
                     discount_percent = 0.0
@@ -2062,7 +2092,7 @@ def response_cost_calculator(
                         getattr(response_object, HIDDEN_PARAMS_ATTR),
                     )
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
-                    if provider_response_cost is not None:
+                    if provider_response_cost is not None and not custom_pricing:
                         return provider_response_cost
 
             response_cost = completion_cost(

litellm/decisions/call.py

@@ -19,11 +19,13 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
 from litellm.types.decisions import (
     DecisionQuestion,
+    DecisionsAudio,
     DecisionsIRRequest,
     DecisionsIRResponse,
     DecisionsJSON,
     DecisionsRequestBody,
     DecisionsResponse,
+    DecisionsVideo,
     OpenAIDecisionInput,
     OpenAIDecisionQuestion,
     OpenAIDecisionRequestBody,
@@ -35,6 +37,8 @@ from litellm.utils import ProviderConfigManager
 
 SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
 OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
+DecisionsAudioClips: TypeAlias = Sequence[DecisionsAudio | Mapping[str, object]]
+DecisionsVideoClips: TypeAlias = Sequence[DecisionsVideo | Mapping[str, object]]
 DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
 
 _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
@@ -49,19 +53,31 @@ _HANDLER: Final = BaseLLMHTTPHandler()
 class SystemOneRequestFields:
     state: DecisionsJSON | None
     questions: SystemOneQuestions | None
+    audio: DecisionsAudioClips | None = None
+    videos: DecisionsVideoClips | None = None
 
     def validate(self, safety_identifier: str | None) -> DecisionsRequestBody:
-        return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": self.state, "questions": self.questions})
+        return _SYSTEMONE_REQUEST_ADAPTER.validate_python(
+            {"state": self.state, "questions": self.questions, "audio": self.audio, "videos": self.videos}
+        )
 
 
 @dataclass(frozen=True, slots=True)
 class OpenAIDecisionRequestFields:
     input: OpenAIDecisionInput | None
     questions: OpenAIDecisionQuestions | None
+    audio: DecisionsAudioClips | None = None
+    videos: DecisionsVideoClips | None = None
 
     def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody:
         return _OPENAI_REQUEST_ADAPTER.validate_python(
-            {"input": self.input, "questions": self.questions, "safety_identifier": safety_identifier}
+            {
+                "input": self.input,
+                "questions": self.questions,
+                "safety_identifier": safety_identifier,
+                "audio": self.audio,
+                "videos": self.videos,
+            }
         )
 
 
@@ -122,15 +138,49 @@ def _drops_params(kwargs: Mapping[str, object]) -> bool:
     return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
 
 
-def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None:
-    if isinstance(safety_identifier, str) or not _drops_params(kwargs):
+def _request_safety_identifier(
+    safety_identifier: object, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object]
+) -> str | None:
+    if not _drops_params(kwargs):
         return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier)
+    if isinstance(safety_identifier, str) and provider_config.supports_safety_identifier:
+        return safety_identifier
     return None
 
 
+def _fields_without_unsupported_media(
+    fields: RequestFields,
+    provider_config: BaseDecisionsConfig,
+    model: str,
+    custom_llm_provider: str,
+    kwargs: Mapping[str, object],
+) -> RequestFields:
+    if not _drops_params(kwargs):
+        return fields
+    return replace(
+        fields,
+        audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
+        videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
+    )
+
+
 @dataclass(frozen=True, slots=True)
-class _UnsupportedSafetyIdentifier:
-    pass
+class _UnsupportedParams:
+    names: tuple[str, ...]
+
+
+def _unsupported_param_names(
+    ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str
+) -> tuple[str, ...]:
+    checks: Final = (
+        (
+            "safety_identifier",
+            ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier,
+        ),
+        ("audio", bool(ir_request.audio) and not provider_config.supports_audio_input(model, custom_llm_provider)),
+        ("videos", bool(ir_request.videos) and not provider_config.supports_video_input(model, custom_llm_provider)),
+    )
+    return tuple(name for name, unsupported in checks if unsupported)
 
 
 def _provider_ir_request(
@@ -138,14 +188,14 @@ def _provider_ir_request(
     *,
     safety_identifier: str | None,
     provider_config: BaseDecisionsConfig,
-    kwargs: Mapping[str, object],
-) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier:
+    model: str,
+    custom_llm_provider: str,
+) -> DecisionsIRRequest | _UnsupportedParams:
     ir_request: Final = _ir_request(request, safety_identifier)
-    if ir_request.safety_identifier is None or provider_config.supports_safety_identifier:
-        return ir_request
-    if _drops_params(kwargs):
-        return replace(ir_request, safety_identifier=None)
-    return _UnsupportedSafetyIdentifier()
+    unsupported: Final = _unsupported_param_names(ir_request, provider_config, model, custom_llm_provider)
+    if unsupported:
+        return _UnsupportedParams(unsupported)
+    return ir_request
 
 
 def prepare_call(
@@ -178,8 +228,10 @@ def prepare_call(
     except ValueError as error:
         raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
     try:
-        request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs)
-        request: Final = fields.validate(request_safety_identifier)
+        request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
+        request: Final = _fields_without_unsupported_media(
+            fields, provider_config, canonical_model, provider, kwargs
+        ).validate(request_safety_identifier)
     except ValidationError as error:
         raise litellm.BadRequestError(
             message=f"Invalid Decisions request: {error}",
@@ -206,12 +258,13 @@ def prepare_call(
         request,
         safety_identifier=request_safety_identifier,
         provider_config=provider_config,
-        kwargs=kwargs,
+        model=canonical_model,
+        custom_llm_provider=provider,
     )
-    if isinstance(ir_request, _UnsupportedSafetyIdentifier):
+    if isinstance(ir_request, _UnsupportedParams):
         raise litellm.UnsupportedParamsError(
             message=(
-                f"{provider} does not support parameters: ['safety_identifier'], for model={model}. "
+                f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}. "
                 "To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
                 "`litellm_settings:\n drop_params: true`\n"
             ),

litellm/decisions/main.py

@@ -6,7 +6,9 @@ import httpx
 from pydantic import TypeAdapter
 
 from litellm.decisions.call import (
+    DecisionsAudioClips,
     DecisionsCall,
+    DecisionsVideoClips,
     OpenAIDecisionQuestions,
     OpenAIDecisionRequestFields,
     RequestFields,
@@ -29,11 +31,16 @@ _OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] =
 def _request_fields(
     decision_input: OpenAIDecisionInput | None,
     questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
+    audio: DecisionsAudioClips | None,
+    videos: DecisionsVideoClips | None,
     kwargs: Mapping[str, object],
 ) -> RequestFields:
     if "state" not in kwargs:
         return OpenAIDecisionRequestFields(
-            input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions)
+            input=decision_input,
+            questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions),
+            audio=audio,
+            videos=videos,
         )
     warnings.warn(
         "litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format",
@@ -43,6 +50,8 @@ def _request_fields(
     return SystemOneRequestFields(
         state=_STATE_ADAPTER.validate_python(kwargs["state"]),
         questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions),
+        audio=audio,
+        videos=videos,
     )
 
 
@@ -52,6 +61,8 @@ def _prepare_decisions_call(
     decision_input: OpenAIDecisionInput | None,
     questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
     safety_identifier: str | None,
+    audio: DecisionsAudioClips | None,
+    videos: DecisionsVideoClips | None,
     api_key: str | None,
     api_base: str | None,
     timeout: float | httpx.Timeout | None,
@@ -61,7 +72,7 @@ def _prepare_decisions_call(
 ) -> DecisionsCall:
     return prepare_call(
         model=model,
-        fields=_request_fields(decision_input, questions, kwargs),
+        fields=_request_fields(decision_input, questions, audio, videos, kwargs),
         safety_identifier=safety_identifier,
         api_key=api_key,
         api_base=api_base,
@@ -83,6 +94,8 @@ async def adecisions(
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: Mapping[str, str] | None = None,
+    audio: DecisionsAudioClips | None = None,
+    videos: DecisionsVideoClips | None = None,
     **kwargs: object,
 ) -> OpenAIDecisionResponse | DecisionsResponse:
     call: Final = _prepare_decisions_call(
@@ -90,6 +103,8 @@ async def adecisions(
         decision_input=input,
         questions=questions,
         safety_identifier=safety_identifier,
+        audio=audio,
+        videos=videos,
         api_key=api_key,
         api_base=api_base,
         timeout=timeout,
@@ -111,6 +126,8 @@ def decisions(
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: Mapping[str, str] | None = None,
+    audio: DecisionsAudioClips | None = None,
+    videos: DecisionsVideoClips | None = None,
     **kwargs: object,
 ) -> OpenAIDecisionResponse | DecisionsResponse:
     call: Final = _prepare_decisions_call(
@@ -118,6 +135,8 @@ def decisions(
         decision_input=input,
         questions=questions,
         safety_identifier=safety_identifier,
+        audio=audio,
+        videos=videos,
         api_key=api_key,
         api_base=api_base,
         timeout=timeout,

litellm/experimental_mcp_client/client.py

@@ -41,6 +41,7 @@ class _SessionCallbackMap(TypedDict):
     sampling_callback: ReadOnly[SamplingFnT | None]
     elicitation_callback: ReadOnly[ElicitationFnT | None]
     logging_callback: ReadOnly[LoggingFnT | None]
+    response_hooks: ReadOnly[tuple[Callable[[httpx2.Response], Awaitable[None]], ...]]
 
 
 from mcp.types import (
@@ -96,6 +97,7 @@ from litellm.constants import (
 )
 from litellm.experimental_mcp_client.tools import list_tools_with_pagination
 from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
+from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx
 from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response
 from litellm.proxy._experimental.mcp_server.result_conversion import (
     age_freshness,
@@ -122,6 +124,20 @@ if TYPE_CHECKING:
     from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback
 
 
+def _mcp_response_hooks() -> tuple[Callable[[httpx2.Response], Awaitable[None]], ...]:
+    if get_active_mcp_request_ctx() is None:
+        return (capture_upstream_error_response,)
+    from litellm.proxy._experimental.mcp_server.legacy_callbacks import (
+        get_scope_response,
+        record_upstream_tool_authorization,
+    )
+
+    return (
+        partial(record_upstream_tool_authorization, pending=get_scope_response()),
+        capture_upstream_error_response,
+    )
+
+
 def to_basic_auth(auth_value: str) -> str:
     """Convert auth value to Basic Auth format."""
     return base64.b64encode(auth_value.encode("utf-8")).decode()
@@ -497,6 +513,7 @@ class MCPClient:
             "sampling_callback": self._sampling_callback,
             "elicitation_callback": self._elicitation_callback,
             "logging_callback": self._logging_callback,
+            "response_hooks": _mcp_response_hooks(),
         }
 
     def session_settings(self) -> tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool]:
@@ -721,7 +738,9 @@ class MCPClient:
                         write_stream,
                         read_timeout_seconds=self.timeout,
                         message_handler=receive_message,
-                        **session_kwargs,
+                        sampling_callback=session_kwargs["sampling_callback"],
+                        elicitation_callback=session_kwargs["elicitation_callback"],
+                        logging_callback=session_kwargs["logging_callback"],
                     )
                     session: Final = await session_ctx.__aenter__()
                     try:
@@ -931,7 +950,14 @@ class MCPClient:
                 verify=ssl_config,
                 follow_redirects=True,
                 event_hooks=MappingProxyType(
-                    {"response": [capture_upstream_error_response], "request": [guard] if guard else []}
+                    {
+                        "response": list(
+                            self._session_callbacks_override["response_hooks"]
+                            if self._session_callbacks_override is not None
+                            else _mcp_response_hooks()
+                        ),
+                        "request": [guard] if guard else [],
+                    }
                 ),
             )
 
@@ -1582,6 +1608,7 @@ class PersistentMCPSession:
         self._initial_callbacks: Final[_SessionCallbackMap] = client.session_callbacks()
         self._current_callbacks: _SessionCallbackMap = self._initial_callbacks
         delegated_callbacks: Final[_SessionCallbackMap] = {
+            "response_hooks": (self._record_response,),
             "sampling_callback": (
                 self._delegate("sampling_callback")
                 if self._initial_callbacks["sampling_callback"] is not None
@@ -1599,6 +1626,10 @@ class PersistentMCPSession:
         client.route_session_callbacks(delegated_callbacks)
         self._task: Final = loop.create_task(self._serve())
 
+    async def _record_response(self, response: httpx2.Response) -> None:
+        for hook in self._current_callbacks["response_hooks"]:
+            await hook(response)
+
     @overload
     def _delegate(self, name: Literal["sampling_callback"]) -> SamplingFnT: ...
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -326,7 +326,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         openai_dialect: Final = (
             stamped_dialect
             if isinstance(stamped_dialect, bool)
-            else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
+            else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
                 model,
                 non_default_params.get("custom_llm_provider"),
                 non_default_params.get("api_base") or non_default_params.get("base_url"),
@@ -384,7 +384,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return model, processed_messages, non_default_params
 
     @staticmethod
-    def _targets_openai_prompt_cache_breakpoint(
+    def targets_openai_prompt_cache_breakpoint(
         model: str | None,
         custom_llm_provider: str | None,
         api_base: object = None,
@@ -786,7 +786,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return AnthropicCacheControlHook._stamped(
             points,
             "_litellm_openai_dialect",
-            AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
+            AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
                 model, custom_llm_provider, api_base, prompt_cache_options
             ),
         )
@@ -1103,7 +1103,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if not injection_points:
             return messages, system
 
-        openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
+        openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
             model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)

litellm/integrations/otel/README.md

@@ -311,10 +311,15 @@ nothing here imports outside it:
   `LITELLM_OTEL_EXCLUDED_SERVICES` (comma-separated) or `excluded_services`
   (a YAML list) under `callback_settings.otel`, naming the datastore services
   to withhold (`redis`, `postgres`, `batch_write_to_db`, `redis_*`, or their
-  `db.system.name` spellings `redis` / `postgresql`). Unknown names are logged
-  as an error and ignored. A span is withheld when its `db.system.name` /
+  `db.system.name` spellings `redis` / `postgresql`). Unknown global names are logged
+  as an error and ignored. Team and key callback values are validated when saved,
+  and invalid values are rejected. A span is withheld when its `db.system.name` /
   `db.system` attribute is in the set, so request root, auth, guardrail and
-  model spans can never be excluded.
+  model spans can never be excluded. Team and key destinations can replace the
+  global set with `excluded_services` in their own `callback_vars`; `"none"`
+  means no datastore spans are excluded, while an omitted value follows the
+  global set. These destination overrides require OTel v2 and are accepted only
+  on `langfuse_otel`, `arize`, `weave_otel` and `newrelic` callbacks.
 - [`baggage.py`](./model/baggage.py) — the single definition of which request-identity
   values are promoted into Baggage (so child spans inherit them) and under which
   attribute keys.

litellm/integrations/otel/model/config.py

@@ -408,6 +408,23 @@ def excluded_db_systems_from(value: object) -> frozenset[str]:
     return _normalize_excluded_services(excluded_service_names(value))
 
 
+def invalid_excluded_service_names(value: str) -> tuple[str, ...]:
+    names: Final = tuple(service.strip().lower() for service in value.split(","))
+    return tuple(
+        name
+        for name in names
+        if name != "none" and (not name or _resolved_db_system_for_excluded_service(name) is None)
+    )
+
+
+def parse_excluded_services(value: object) -> frozenset[str] | None:
+    if not isinstance(value, str):
+        return None
+    if value.strip().lower() == "none":
+        return frozenset()
+    return excluded_db_systems_from(value)
+
+
 def parse_capture_message_content(value: object) -> str | None:
     return value if isinstance(value, str) and value in CAPTURE_MESSAGE_CONTENT_VALUES else None
 
@@ -448,9 +465,13 @@ def _normalize_excluded_services(services: frozenset[str]) -> frozenset[str]:
 
 
 def _db_system_for_excluded_service(service: str) -> str | None:
-    resolved: Final = db_system(service) if service != POSTGRESQL else POSTGRESQL
+    resolved: Final = _resolved_db_system_for_excluded_service(service)
     if resolved is None:
         verbose_logger.error(
             "excluded_services: %r is not a datastore service; ignored. Allowed: postgres, redis", service
         )
     return resolved
+
+
+def _resolved_db_system_for_excluded_service(service: str) -> str | None:
+    return db_system(service) if service != POSTGRESQL else POSTGRESQL

litellm/integrations/otel/model/destination.py

@@ -57,6 +57,10 @@ class OtelDestination(LiteLLMBaseModel):
             "Omitted follows the global setting."
         ),
     )
+    excluded_db_systems: frozenset[str] | None = Field(
+        default=None,
+        description="An explicit set replaces the global database-span exclusions; ``None`` follows the global set.",
+    )
 
     def captures_content(self, default: str) -> bool:
         setting: Final = self.capture_message_content
@@ -75,9 +79,10 @@ class OtelDestination(LiteLLMBaseModel):
     def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]:
         """Identity for processor reuse, so one destination means one exporter.
 
-        ``span_scope``, the sampling rates and ``capture_message_content`` are left out on
-        purpose: they decide which spans reach the processor and what they carry, not how
-        the processor exports them, so two views of the same account share one exporter.
+        ``span_scope``, the sampling rates, ``capture_message_content`` and
+        ``excluded_db_systems`` are left out on purpose: they decide which spans reach
+        the processor and what they carry, not how the processor exports them, so two
+        views of the same account share one exporter.
         """
         return (
             self.endpoint,

litellm/integrations/otel/plumbing/providers.py

@@ -705,7 +705,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
             if (
                 self._operator_already_writes(span, destination, suppressed)
                 or not _in_scope(span, destination.span_scope)
-                or _is_excluded_database_span(attributes, self._excluded_db_systems)
+                or _is_excluded_database_span(
+                    attributes,
+                    self._excluded_db_systems
+                    if destination.excluded_db_systems is None
+                    else destination.excluded_db_systems,
+                )
             ):
                 continue
             if not _is_sampled(destination):

litellm/integrations/otel/presets/destinations.py

@@ -188,6 +188,7 @@ def destination_for(
     params: StandardCallbackDynamicParams,
     service_name: str | None = None,
     capture_message_content: str | None = None,
+    excluded_db_systems: frozenset[str] | None = None,
 ) -> OtelDestination | None:
     """The destination ``params`` names for ``callback_name``, or ``None``.
 
@@ -219,6 +220,7 @@ def destination_for(
         protocol=protocol,
         span_scope=_span_scope(callback_name, params),
         capture_message_content=capture_message_content,
+        excluded_db_systems=excluded_db_systems,
         success_sampling_rate=success_sampling_rate,
         error_sampling_rate=error_sampling_rate,
     )

litellm/litellm_core_utils/cost_discount.py

@@ -0,0 +1,83 @@
+import fnmatch
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Final
+
+_GLOB_CHARS: Final = frozenset("*?[")
+_GLOB_TOKEN: Final = re.compile(
+    r"\[!\][^\]]*\]|\[\][^\]]*\]|\[!(?!\])[^\]]*\]|\[(?![!\]])[^\]]*\]|.",
+    re.DOTALL,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class CostDiscountKey:
+    provider: str
+    model_pattern: str | None
+
+
+def parse_cost_discount_key(key: str) -> CostDiscountKey:
+    provider, separator, pattern = key.partition("/")
+    return CostDiscountKey(provider=provider, model_pattern=pattern if separator else None)
+
+
+def _literal_length(pattern: str) -> int:
+    return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?")
+
+
+def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]:
+    prefix_run: Final = re.match(f"(?:{re.escape(provider_prefix)})*", model)
+    depth: Final = len(prefix_run.group()) // len(provider_prefix) if prefix_run else 0
+    if depth == 0:
+        return (model,)
+    return tuple(model[len(provider_prefix) * level :] for level in range(1, depth + 1))
+
+
+def resolve_cost_discount(
+    cost_discount_config: Mapping[str, float],
+    custom_llm_provider: str | None,
+    model: str | None,
+    region_name: str | None = None,
+) -> float | None:
+    if not custom_llm_provider:
+        return None
+
+    provider_prefix: Final = f"{custom_llm_provider}/"
+    region_prefix: Final = f"{provider_prefix}{region_name}/" if region_name else ""
+    model_names: Final = (
+        ()
+        if model is None
+        else _model_names(model, provider_prefix)
+        + (
+            _model_names(f"{provider_prefix}{model[len(region_prefix) :]}", provider_prefix)
+            if region_prefix and model.startswith(region_prefix)
+            else ()
+        )
+    )
+    patterns: Final = tuple(
+        parsed.model_pattern
+        for parsed in (parse_cost_discount_key(key) for key in cost_discount_config)
+        if parsed.provider == custom_llm_provider and parsed.model_pattern is not None
+    )
+
+    exact_patterns: Final = frozenset(pattern for pattern in patterns if _GLOB_CHARS.isdisjoint(pattern))
+    exact: Final = next((name for name in model_names if name in exact_patterns), None)
+    if exact is not None:
+        return cost_discount_config[f"{custom_llm_provider}/{exact}"]
+    prefixed_markers: Final = (provider_prefix,) if not region_name else (provider_prefix, f"{region_name}/")
+    matches: Final = tuple(
+        pattern
+        for pattern in patterns
+        if not _GLOB_CHARS.isdisjoint(pattern)
+        and any(
+            fnmatch.fnmatchcase(name, pattern)
+            for name in model_names
+            if "/" in pattern or not name.startswith(prefixed_markers)
+        )
+    )
+    if matches:
+        best_match: Final = max(matches, key=_literal_length)
+        return cost_discount_config[f"{custom_llm_provider}/{best_match}"]
+
+    return cost_discount_config.get(custom_llm_provider)

litellm/litellm_core_utils/decisions_pricing.py

@@ -0,0 +1,33 @@
+"""Decisions-specific pricing read from ``<field>_decisions`` cost map fields.
+
+A model that serves both chat and Decisions keeps one cost map row. Rates that
+differ for ``/v1/decisions`` and ``/v1/systemone`` live on that row under the
+same field name with a ``_decisions`` suffix, e.g. ``output_cost_per_token_decisions``,
+and override the plain field for those two routes only. A row without any such
+field bills Decisions calls from its plain fields.
+"""
+
+from collections.abc import Mapping
+from typing import Final
+
+DECISIONS_PRICING_SUFFIX: Final = "_decisions"
+DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
+
+
+def is_decisions_call_type(call_type: str) -> bool:
+    return call_type in DECISIONS_CALL_TYPES
+
+
+def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
+    """Return ``{plain_field: value}`` for every ``<plain_field>_decisions`` field on ``row``."""
+    return {
+        key.removesuffix(DECISIONS_PRICING_SUFFIX): value
+        for key, value in row.items()
+        if key.endswith(DECISIONS_PRICING_SUFFIX)
+    }
+
+
+def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
+    """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
+    plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}
+    return {**plain, **decisions_pricing_fields(row)}

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -880,21 +880,37 @@ def _map_openai_like_exception(
 
 
 _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
+_BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile(
+    r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']"""
+)
+_BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
+    "prompt is too long: your prompt exceeds the model's context window"
+)
+
+
+def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
+    original_response: Final = getattr(original_exception, "response", None)
+    if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
+        return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
+    if original_response.status_code == 400:
+        return original_response
+    return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
 
 
 def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
     """
-    Mantle reports context overflow as a structured validation error rather than
-    the plain-text patterns Bedrock itself uses, so it needs its own detection and a
-    message clients recognize as context overflow (litellm/litellm#36546).
+    Mantle reports context overflow as a validation_error carrying the token counts, or as
+    OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event
+    with no error type. Clients such as Claude Code only treat "prompt is too long" as
+    overflow (litellm/litellm#36546).
     """
-    if "invalid_request_error" not in error_str and "validation_error" not in error_str:
-        return None
     match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
-    if match is None:
-        return None
-    prompt_tokens, max_tokens = match.groups()
-    return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
+    if match is not None:
+        prompt_tokens, max_tokens = match.groups()
+        return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
+    if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None:
+        return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE
+    return None
 
 
 def _map_bedrock_exception(
@@ -914,7 +930,7 @@ def _map_bedrock_exception(
                 message=mantle_context_window_message,
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=_bedrock_mantle_context_window_response(original_exception),
             )
     if (
         "too many tokens" in error_str

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -99,6 +99,18 @@ def validate_capture_message_content_value(value: str) -> None:
         )
 
 
+def validate_excluded_services_value(value: str) -> None:
+    from litellm.integrations.otel.model.config import invalid_excluded_service_names
+
+    names: Final = tuple(name.strip().lower() for name in value.split(","))
+    invalid_names: Final = invalid_excluded_service_names(value)
+    allowed: Final = "redis, postgres, postgresql, batch_write_to_db, redis_-prefixed service names, or none"
+    if invalid_names or ("none" in names and len(names) != 1):
+        raise ValueError(
+            f"Invalid excluded_services {value!r}: use {allowed}; 'none' must be used alone and names cannot be empty"
+        )
+
+
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
 _supported_callback_params: Final[tuple[str, ...]] = (
     "langfuse_public_key",

litellm/litellm_core_utils/litellm_logging.py

@@ -5,6 +5,7 @@ import copy
 import datetime
 import functools
 import json
+import math
 import os
 import re
 import subprocess
@@ -732,6 +733,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # Initialize cost breakdown field
         self.cost_breakdown: CostBreakdown | None = None
         self.billed_token_rates: BilledTokenRates | None = None
+        self._provider_reported_cost: float | None = None
 
         # Init Caching related details
         self.caching_details: CachingDetails | None = None
@@ -1948,7 +1950,16 @@ class Logging(LiteLLMLoggingBaseClass):
             return None
 
         try:
-            response_cost: Final = litellm.response_cost_calculator(**response_cost_calculator_kwargs)
+            reported_cost: Final = self._provider_reported_cost
+            response_cost: Final = (
+                reported_cost
+                if isinstance(reported_cost, float)
+                and math.isfinite(reported_cost)
+                and reported_cost >= 0
+                and not custom_pricing
+                and base_model is None
+                else litellm.response_cost_calculator(**response_cost_calculator_kwargs)
+            )
 
             verbose_logger.debug("response_cost: %s", response_cost)
             additional_response_cost: Final[object] = self.model_call_details.get("additional_response_cost")

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1269,6 +1269,13 @@ def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) ->
     return float(provider_specific_entry.get(inference_geo.lower(), 1.0))
 
 
+def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float:
+    provider_specific_entry: Final = model_info.get("provider_specific_entry")
+    if getattr(usage, "speed", None) != "fast" or not provider_specific_entry:
+        return 1.0
+    return float(provider_specific_entry.get("fast", 1.0))
+
+
 def _resolve_reasoning_token_cost(
     model_info: ModelInfo,
     service_tier: str | None,
@@ -1633,6 +1640,7 @@ def _cost_map_billed_rates(
         get_regional_uplift_multiplier(model_info, data_residency)
         * get_vertex_regional_endpoint_uplift(model_info, vertex_location)
         * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
+        * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
     )
     return BilledTokenRates(
         input_cost_per_token=prompt_base_cost,

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -29,6 +29,7 @@ from litellm.types.llms.openai import (
     ChatCompletionThinkingBlock,
     ChatCompletionToolParam,
     ChatCompletionUserMessage,
+    PromptCacheBreakpoint,
 )
 from litellm.types.utils import (
     Choices,
@@ -1793,6 +1794,16 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
     return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key
 
 
+def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
+    """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
+    if block.get("cache_control") is None:
+        return block
+    unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"}
+    return cast(  # cast-ok: same block shape as the input with the marker key renamed
+        _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit"))
+    )
+
+
 LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"})
 
 

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -5093,7 +5093,7 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
 def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockToolBlock | None:
     from litellm.llms.bedrock.common_utils import (
         bedrock_model_accepts_cache_points,
-        is_claude_4_5_on_bedrock,
+        supports_bedrock_prompt_cache_ttl,
     )
 
     cache_control: Final = tool.get("cache_control", None)
@@ -5103,7 +5103,7 @@ def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockT
             cache_point_block: Final[CachePointBlock] = {"type": "default"}
             if isinstance(cache_control, dict) and "ttl" in cache_control:
                 ttl: Final = cache_control["ttl"]
-                if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model):
+                if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model):
                     cache_point_block["ttl"] = ttl
             return {"cachePoint": cache_point_block}
     return None

litellm/llms/__init__.py

@@ -62,6 +62,15 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
         )
 
         return cost_per_web_search_request_vertex_ai(usage=usage, model_info=model_info)
+    elif custom_llm_provider == "azure_ai":
+        from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
+
+        azure_ai_model_key: Final[str] = model_info.get("key", "") if model_info else ""
+        if "claude" not in azure_ai_model_key.lower() and get_web_search_requests_from_usage(usage) is None:
+            return None
+        from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
+
+        return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
     elif custom_llm_provider == "perplexity":
         # Perplexity handles search costs internally in its own cost calculator
         # Return 0.0 to indicate costs are already accounted for

litellm/llms/anthropic/cost_calculation.py

@@ -10,6 +10,7 @@ from pydantic import ValidationError
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
     generic_cost_per_token,
     get_provider_specific_geo_multiplier,
+    get_provider_specific_speed_multiplier,
     get_web_search_requests_from_usage,
 )
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -50,14 +51,8 @@ def cost_per_token(
             if model_info is not None
             else litellm.get_model_info(model=model, custom_llm_provider="anthropic")
         )
-        provider_specific_entry: Final = effective_info.get("provider_specific_entry")
-
         geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage)
-        speed_multiplier: Final = (
-            provider_specific_entry.get("fast", 1.0)
-            if provider_specific_entry and getattr(usage, "speed", None) == "fast"
-            else 1.0
-        )
+        speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage)
 
         if speed_multiplier != 1.0:
             prompt_cost *= speed_multiplier

litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py

@@ -19,7 +19,7 @@ from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
-from litellm.exceptions import MidStreamFallbackError
+from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
 from litellm.litellm_core_utils.hidden_params import set_hidden_params
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.anthropic import (
@@ -64,7 +64,7 @@ def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]:
 
 
 def _error_status_and_message(exc: Exception) -> tuple[int, str]:
-    if isinstance(exc, (BaseLLMException, MidStreamFallbackError)):
+    if isinstance(exc, (BaseLLMException, MidStreamFallbackError, ContextWindowExceededError)):
         return exc.status_code, exc.message
     return 500, str(exc) or "Upstream stream ended before completion"
 

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -4,10 +4,14 @@ Handler for the Anthropic v1/messages -> OpenAI Responses API path.
 Used when the target model is an OpenAI or Azure model.
 """
 
-from collections.abc import AsyncIterator, Coroutine, Mapping
+from collections.abc import AsyncIterator, Coroutine, Mapping, Sequence
 from typing import Final, TypeAlias
 
+from pydantic import TypeAdapter, ValidationError
+
 import litellm
+from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
 from litellm.types.llms.anthropic import (
     AllAnthropicMessageValues,
@@ -27,6 +31,7 @@ from .streaming_iterator import AnthropicResponsesStreamWrapper
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
 AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
+_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
 
 _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
 
@@ -45,6 +50,24 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
     return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
 
 
+def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
+    provider: Final = forwarded_kwargs.get("custom_llm_provider")
+    return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
+        model,
+        provider if isinstance(provider, str) else None,
+        forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
+        forwarded_kwargs.get("prompt_cache_options"),
+    )
+
+
+def _blocks_with_prompt_cache_breakpoints(content: object) -> Sequence[Mapping[str, object]]:
+    """The content's blocks with each ``cache_control`` mark rewritten, empty when it is not a list of blocks."""
+    try:
+        return [cache_control_as_prompt_cache_breakpoint(block) for block in _BLOCK_LIST.validate_python(content)]
+    except ValidationError:
+        return ()
+
+
 def _build_responses_kwargs(
     *,
     max_tokens: int,
@@ -68,10 +91,28 @@ def _build_responses_kwargs(
     """
     Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
     """
+    forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
+    reads_prompt_cache_breakpoint: Final = _reads_prompt_cache_breakpoint(model, forwarded_kwargs)
+    request_messages: Final[AnthropicRequestMessages] = (
+        [
+            {
+                **message,
+                "content": list(_blocks_with_prompt_cache_breakpoints(message.get("content")))
+                or message.get("content"),
+            }
+            for message in messages
+        ]
+        if reads_prompt_cache_breakpoint
+        else messages
+    )
+    request_system: Final = (
+        (list(_blocks_with_prompt_cache_breakpoints(system)) or system) if reads_prompt_cache_breakpoint else system
+    )
+
     # Build a typed AnthropicMessagesRequest for the adapter
     request_data: Final[AnthropicMessagesRequest] = {
         "model": model,
-        "messages": messages,
+        "messages": request_messages,
         "max_tokens": max_tokens,
     }
     if context_management:
@@ -80,8 +121,8 @@ def _build_responses_kwargs(
         request_data["output_config"] = output_config
     if metadata:
         request_data["metadata"] = metadata
-    if system:
-        request_data["system"] = system
+    if request_system:
+        request_data["system"] = request_system
     if temperature is not None:
         request_data["temperature"] = temperature
     if thinking:
@@ -96,7 +137,6 @@ def _build_responses_kwargs(
         request_data["output_format"] = output_format
 
     anthropic_request: Final = AnthropicMessagesRequest(**request_data)
-    forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
     responses_kwargs: Final = _ADAPTER.translate_request(
         anthropic_request,
         include_encrypted_reasoning=_provider_returns_encrypted_reasoning(

litellm/llms/anthropic/pass_through/responses_adapters/transformation.py

@@ -131,12 +131,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
     def _tool_result_output_value(
         output_text: str,
         file_parts: tuple[dict[str, str], ...],  # mutable-ok: json content parts
+        tool_result: Mapping[str, object],
     ) -> str | list[dict[str, str]]:  # mutable-ok: API message payload
-        """Plain string output, or a part list when document file parts are present."""
-        if not file_parts:
-            return output_text
+        """Plain string output, or a part list when document file parts or a cache breakpoint are present.
+
+        OpenAI reads a breakpoint only on a part, so a marked result ends on a part that carries it.
+        """
+        prompt_cache_breakpoint: Final = tool_result.get("prompt_cache_breakpoint")
         text_parts: Final = [{"type": "input_text", "text": output_text}] if output_text else []
-        return [*text_parts, *file_parts]
+        parts: Final = [*text_parts, *file_parts]
+        if not parts or (not file_parts and prompt_cache_breakpoint is None):
+            return output_text
+        return [*parts[:-1], with_prompt_cache_breakpoint(parts[-1], prompt_cache_breakpoint)]
 
     @staticmethod
     def _translate_midturn_system_content_to_responses(
@@ -327,7 +333,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                 {
                                     "type": "function_call_output",
                                     "call_id": tool_use_id,
-                                    "output": self._tool_result_output_value(output_text, tool_file_parts),
+                                    "output": self._tool_result_output_value(output_text, tool_file_parts, block),
                                 }
                             )
                     if tool_image_parts:

litellm/llms/base_llm/decisions/transformation.py

@@ -8,6 +8,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 from typing_extensions import assert_never
 
+import litellm
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.decisions import (
@@ -47,6 +48,7 @@ from litellm.types.decisions import (
     UnsupportedDecisionsRequest,
     systemone_choice_key,
 )
+from litellm.types.utils import is_decisions_model_mode
 
 _PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
 _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
@@ -100,6 +102,8 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion:
 def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest:
     return DecisionsIRRequest(
         input=DecisionsIRState(state=request.state),
+        audio=request.audio,
+        videos=request.videos,
         questions=tuple(_ir_question(name, question) for name, question in request.questions.items()),
     )
 
@@ -166,8 +170,12 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
 
 
 def ir_to_systemone_request(
-    model: str, request: DecisionsIRRequest
+    model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
 ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+    if request.audio and not supports_audio:
+        return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
+    if request.videos and not supports_video:
+        return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
     state: Final = _systemone_state(request.input)
     if isinstance(state, UnsupportedDecisionsRequest):
         return state
@@ -175,6 +183,16 @@ def ir_to_systemone_request(
     return {
         "model": model,
         "state": state,
+        **(
+            {"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]}
+            if request.audio
+            else {}
+        ),
+        **(
+            {"videos": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.videos]}
+            if request.videos
+            else {}
+        ),
         "questions": {key: _systemone_question(question) for key, question in keyed_questions},
     }
 
@@ -296,6 +314,14 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
     )
 
 
+def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
+    try:
+        model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
+    except Exception:  # noqa: BLE001  # get_model_info raises bare Exception for an unmapped model, which has no mode
+        return False
+    return is_decisions_model_mode(model_mode)
+
+
 class BaseDecisionsConfig(ABC):
     path: str = "/v1/systemone"
     api_key_env: tuple[str, ...] = ()
@@ -355,13 +381,28 @@ class BaseDecisionsConfig(ABC):
     ) -> tuple[Mapping[str, str], bytes | None]:
         return headers, None
 
+    def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
+        return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
+
+    def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
+        return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
+
     def transform_decisions_request(
         self,
         model: str,
         request: DecisionsIRRequest,
         custom_llm_provider: str,
     ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
-        return ir_to_systemone_request(self.request_model(model), request)
+        return ir_to_systemone_request(
+            self.request_model(model),
+            request,
+            supports_audio=self.supports_audio_input(model, custom_llm_provider),
+            supports_video=self.supports_video_input(model, custom_llm_provider),
+        )
 
     def unwrap_response(self, payload: object) -> object:
         return payload

litellm/llms/bedrock/chat/converse_transformation.py

@@ -110,8 +110,8 @@ from ..common_utils import (
     get_anthropic_beta_from_headers,
     get_bedrock_tool_name,
     is_bedrock_application_inference_profile_arn,
-    is_claude_4_5_on_bedrock,
     normalize_bedrock_opus_output_config_effort,
+    supports_bedrock_prompt_cache_ttl,
 )
 
 if TYPE_CHECKING:
@@ -1385,7 +1385,7 @@ class AmazonConverseConfig(BaseConfig):
         cache_point: Final = CachePointBlock(type="default")
         if isinstance(control, dict) and "ttl" in control:
             ttl: Final = control["ttl"]
-            if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model):
+            if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model):
                 cache_point["ttl"] = ttl
         return cache_point
 

litellm/llms/bedrock/common_utils.py

@@ -9,6 +9,7 @@ import functools
 import json
 import os
 import re
+import warnings
 from collections.abc import Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@@ -1230,15 +1231,11 @@ def bedrock_supports_tool_search(model: str) -> bool:
     return AnthropicModelInfo.supports_model_capability(model, "supports_tool_search", "bedrock")
 
 
-def is_claude_4_5_on_bedrock(model: str) -> bool:
+def supports_bedrock_prompt_cache_ttl(model: str) -> bool:
     """
-    Check if the model supports Bedrock prompt caching with an extended '1h' TTL
-    (in addition to the default 5m TTL).
-
-    Backed by the ``cache_creation_input_token_cost_above_1hr`` field in
-    ``model_prices_and_context_window.json`` instead of a hardcoded list of
-    model-name patterns, so newly released models pick up support as soon as
-    their pricing entry ships, with no code change required here.
+    Whether Bedrock accepts a ``ttl`` ("5m"/"1h") on prompt ``cache_control``
+    for this model, decided by ``cache_creation_input_token_cost_above_1hr`` on
+    the model's or its base model's cost-map entry.
     """
     return any(
         (litellm.model_cost.get(candidate) or {}).get("cache_creation_input_token_cost_above_1hr") is not None
@@ -1246,6 +1243,15 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
     )
 
 
+def is_claude_4_5_on_bedrock(model: str) -> bool:
+    warnings.warn(
+        "is_claude_4_5_on_bedrock is deprecated, use supports_bedrock_prompt_cache_ttl",
+        UserWarning,
+        stacklevel=2,
+    )
+    return supports_bedrock_prompt_cache_ttl(model)
+
+
 _BEDROCK_MODEL_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
 

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -34,11 +34,11 @@ from litellm.llms.bedrock.common_utils import (
     bedrock_supports_tool_search,
     ensure_bedrock_anthropic_messages_tool_names,
     get_anthropic_beta_from_headers,
-    is_claude_4_5_on_bedrock,
     normalize_bedrock_opus_output_config_effort,
     normalize_custom_field_on_tools,
     normalize_tool_input_schema_types_for_bedrock_invoke,
     strip_unsupported_bedrock_invoke_output_config_keys,
+    supports_bedrock_prompt_cache_ttl,
     tools_without_eager_input_streaming,
 )
 from litellm.llms.bedrock.request_metadata import (
@@ -169,9 +169,9 @@ class AmazonAnthropicClaudeMessagesConfig(
             anthropic_messages_request: The request dictionary to modify in-place
             model: The model name to check if it supports ttl
         """
-        is_claude_4_5 = False
+        supports_ttl = False
         if model:
-            is_claude_4_5 = self._is_claude_4_5_on_bedrock(model)
+            supports_ttl = self._supports_prompt_cache_ttl(model)
 
         def _sanitize_cache_control(cache_control: dict) -> None:
             if not isinstance(cache_control, dict):
@@ -181,7 +181,7 @@ class AmazonAnthropicClaudeMessagesConfig(
             # Remove ttl for models that don't support it
             if "ttl" in cache_control:
                 ttl: Final = cache_control["ttl"]
-                if is_claude_4_5 and ttl in ["5m", "1h"]:
+                if supports_ttl and ttl in ["5m", "1h"]:
                     return
                 cache_control.pop("ttl", None)
 
@@ -386,19 +386,9 @@ class AmazonAnthropicClaudeMessagesConfig(
         ]
         return any(pattern in model_lower for pattern in opus_4_5_patterns)
 
-    def _is_claude_4_5_on_bedrock(self, model: str) -> bool:
-        """
-        Check if the model is Claude 4.5 on Bedrock.
-
-        Claude Sonnet 4.5, Haiku 4.5, and Opus 4.5 support 1-hour prompt caching.
-
-        Args:
-            model: The model name
-
-        Returns:
-            True if the model is Claude 4.5
-        """
-        return is_claude_4_5_on_bedrock(model)
+    def _supports_prompt_cache_ttl(self, model: str) -> bool:
+        """Whether Bedrock accepts a ``ttl`` on prompt ``cache_control`` for ``model``."""
+        return supports_bedrock_prompt_cache_ttl(model)
 
     def _supports_tool_search_on_bedrock(self, model: str) -> bool:
         """

litellm/llms/databricks/chat/transformation.py

@@ -285,6 +285,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             "tool_choice",
             "reasoning_effort",
             "thinking",
+            "service_tier",
         ]
 
     @staticmethod
@@ -697,6 +698,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         model_response.id = completion_response["id"]
         model_response.created = completion_response["created"]
         setattr(model_response, "usage", Usage(**completion_response["usage"]))
+        served_tier: Final = completion_response.get("service_tier")
+        if isinstance(served_tier, str) and served_tier:
+            setattr(model_response, "service_tier", served_tier)  # noqa: B010  # no declared field
 
         model_response.choices = self._transform_dbrx_choices(
             choices=completion_response["choices"],

litellm/llms/fireworks_ai/decisions/transformation.py

@@ -0,0 +1,20 @@
+from dataclasses import replace
+from typing import Final
+
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, parse_systemone_response
+from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
+
+
+class FireworksAIDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("FIREWORKS_API_KEY", "FIREWORKS_AI_API_KEY")
+    api_base_env = ("FIREWORKS_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.fireworks.ai/inference/v1"
+
+    def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+        response: Final = parse_systemone_response(self.unwrap_response(payload), request)
+        cached: Final = response.usage.extra.get("cached_input_tokens")
+        if isinstance(cached, bool) or not isinstance(cached, int):
+            return response
+        return replace(response, usage=replace(response.usage, cached_tokens=cached))

litellm/llms/openai/decisions/transformation.py

@@ -88,6 +88,8 @@ def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion:
 def openai_request_to_ir(request: OpenAIDecisionRequestBody) -> DecisionsIRRequest:
     return DecisionsIRRequest(
         input=_ir_input(request.input),
+        audio=request.audio,
+        videos=request.videos,
         questions=tuple(_ir_question(question) for question in request.questions),
         safety_identifier=request.safety_identifier,
     )
@@ -164,6 +166,8 @@ def _has_one_option(question: DecisionsIRQuestion) -> bool:
 
 
 def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+    if request.audio or request.videos:
+        return UnsupportedDecisionsRequest(reason="audio/video input is not supported by this Decisions provider")
     if any(_has_one_option(question) for question in request.questions):
         return _SINGLE_OPTION
     return {
@@ -309,6 +313,12 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
     def resolve_api_key(self, api_key: str | None) -> str | None:
         return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
 
+    def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
+        return False
+
+    def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
+        return False
+
     def transform_decisions_request(
         self,
         model: str,

litellm/llms/openai_like/providers.json

@@ -180,6 +180,12 @@
     "api_key_env": "COGNITION_API_KEY",
     "api_base_env": "COGNITION_API_BASE"
   },
+  "coralbricks": {
+    "base_url": "https://inference.coralbricks.ai/v1",
+    "api_key_env": "CORALBRICKS_API_KEY",
+    "api_base_env": "CORALBRICKS_API_BASE",
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
+  },
   "cortecs": {
     "base_url": "https://api.cortecs.ai/v1",
     "api_key_env": "CORTECS_API_KEY",

litellm/llms/pass_through/typesafe_logging_handler.py

@@ -6,6 +6,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 
 import litellm
+from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import (
     get_standard_logging_object_payload,  # pyright: ignore[reportUnknownVariableType]  # legacy helper has an untyped signature
@@ -51,7 +52,9 @@ def _pricing_for(model_keys: tuple[str, ...]) -> _RegistryPricing:
             continue
         try:
             return _REGISTRY_PRICING_ADAPTER.validate_python(
-                litellm.model_cost[model_key]  # pyright: ignore[reportUnknownMemberType]  # registry is dynamically typed
+                apply_decisions_pricing(
+                    litellm.model_cost[model_key]  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # registry is dynamically typed
+                )
             )
         except ValidationError:
             continue

litellm/llms/sagemaker/common_utils.py

@@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import Final
 
 import httpx
-from pydantic import ConfigDict, RootModel, TypeAdapter, with_config
+from pydantic import BaseModel, ConfigDict, Field, RootModel, TypeAdapter, ValidationError, with_config
 from typing_extensions import ReadOnly, Required, TypedDict
 
 import litellm
@@ -13,7 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.utils import GenericStreamingChunk as GChunk
 from litellm.types.utils import StreamingChatCompletionChunk
 
-_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 
 
 @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@@ -54,7 +54,9 @@ def _load_sagemaker_response_stream_shape():
         from botocore.model import ServiceModel
 
         loader: Final = Loader()
-        service_dict: Final = _JSON_OBJECT.validate_python(loader.load_service_model("sagemaker-runtime", "service-2"))
+        service_dict: Final = STR_KEYED_MAPPING.validate_python(
+            loader.load_service_model("sagemaker-runtime", "service-2")
+        )
         return ServiceModel(service_dict).shape_for("InvokeEndpointWithResponseStreamOutput")
     except Exception as e:
         verbose_logger.warning(
@@ -76,6 +78,38 @@ def get_sagemaker_response_stream_shape():
     return _load_sagemaker_response_stream_shape()
 
 
+def with_inference_component_hint(error_message: str) -> str:
+    if "Inference Component Name header is required" in error_message:
+        return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"
+    if "Inference Component Name header is not allowed" in error_message:
+        return f"{error_message}\n remove `model_id` from this deployment, the endpoint has no inference components"
+    return error_message
+
+
+class _ClientErrorBody(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+    message: str | None = Field(default=None, alias="Message")
+
+
+class _ClientErrorMetadata(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+    http_status_code: int = Field(default=500, alias="HTTPStatusCode")
+
+
+class _ClientErrorResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+    error: _ClientErrorBody = Field(default_factory=_ClientErrorBody, alias="Error")
+    response_metadata: _ClientErrorMetadata = Field(default_factory=_ClientErrorMetadata, alias="ResponseMetadata")
+
+
+def client_error_details(error: Exception) -> tuple[int, str]:
+    try:
+        response: Final = _ClientErrorResponse.model_validate(getattr(error, "response", None))
+    except ValidationError:
+        return 500, str(error)
+    return response.response_metadata.http_status_code, response.error.message or str(error)
+
+
 class SagemakerError(BaseLLMException):
     def __init__(
         self,

litellm/llms/sagemaker/completion/handler.py

@@ -4,6 +4,7 @@ from copy import deepcopy
 from typing import Final, cast
 
 import httpx
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -24,12 +25,26 @@ from litellm.utils import (
     get_secret,
 )
 
-from ..common_utils import AWSEventStreamDecoder, SagemakerError
+from ..common_utils import (
+    STR_KEYED_MAPPING,
+    AWSEventStreamDecoder,
+    SagemakerError,
+    client_error_details,
+    with_inference_component_hint,
+)
 from ..embedding.transformation import SagemakerEmbeddingConfig
 from .transformation import SagemakerConfig
 
 sagemaker_config: Final = SagemakerConfig()
 
+
+class _InvokeEndpointArgs(TypedDict):
+    EndpointName: ReadOnly[str]
+    ContentType: ReadOnly[str]
+    Body: ReadOnly[bytes]
+    CustomAttributes: ReadOnly[str]
+
+
 """
 SAGEMAKER AUTH Keys/Vars
 os.environ['AWS_ACCESS_KEY_ID'] = ""
@@ -288,10 +303,7 @@ class SagemakerLLM(BaseAWSLLM):
                 raise e
         except Exception as e:
             verbose_logger.error("Sagemaker error %s", str(e))
-            status_code: Final = getattr(e, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode", 500)
-            error_message = getattr(e, "response", {}).get("Error", {}).get("Message", str(e))
-            if "Inference Component Name header is required" in error_message:
-                error_message += "\n pass in via `litellm.completion(..., model_id={InferenceComponentName})`"
+            status_code, error_message = client_error_details(e)
             raise SagemakerError(status_code=status_code, message=error_message)
 
         return sagemaker_config.transform_response(
@@ -512,10 +524,7 @@ class SagemakerLLM(BaseAWSLLM):
                 )
                 raise e
         except Exception as e:
-            error_message = f"{e}"
-            if "Inference Component Name header is required" in error_message:
-                error_message += "\n pass in via `litellm.completion(..., model_id={InferenceComponentName})`"
-            raise SagemakerError(status_code=500, message=error_message)
+            raise SagemakerError(status_code=500, message=str(e))
         return sagemaker_config.transform_response(
             model=model,
             raw_response=response,
@@ -574,16 +583,29 @@ class SagemakerLLM(BaseAWSLLM):
         #### EMBEDDING LOGIC
         # Transform request based on model type
         provider_config: Final = SagemakerEmbeddingConfig.get_model_config(model)
-        request_data: Final = provider_config.transform_embedding_request(model, input, optional_params, {})
+        endpoint_name: Final = SagemakerEmbeddingConfig.get_endpoint_name(model)
+        embedding_params: Final = STR_KEYED_MAPPING.validate_python(optional_params)
+        inference_component_name: Final = embedding_params.get("model_id")
+        if inference_component_name is not None and not isinstance(inference_component_name, str):
+            raise SagemakerError(status_code=400, message="model_id must be the inference component name as a string")
+        body_params: Final = {k: v for k, v in embedding_params.items() if k != "model_id"}
+        request_data: Final = provider_config.transform_embedding_request(endpoint_name, input, body_params, {})
         data: Final = json.dumps(request_data).encode("utf-8")
+        invoke_args: Final = _InvokeEndpointArgs(
+            EndpointName=endpoint_name,
+            ContentType="application/json",
+            Body=data,
+            CustomAttributes="accept_eula=true",
+        )
 
         ## LOGGING
         request_str: Final = f"""
         response = client.invoke_endpoint(
-            EndpointName={model},
+            EndpointName={endpoint_name},
             ContentType="application/json",
             Body=f"{data!r}",  # Use !r for safe representation
             CustomAttributes="accept_eula=true",
+            InferenceComponentName={inference_component_name},
         )"""
         logging_obj.pre_call(
             input=input,
@@ -592,16 +614,14 @@ class SagemakerLLM(BaseAWSLLM):
         )
         ## EMBEDDING CALL
         try:
-            response = client.invoke_endpoint(
-                EndpointName=model,
-                ContentType="application/json",
-                Body=data,
-                CustomAttributes="accept_eula=true",
+            response = (
+                client.invoke_endpoint(**invoke_args, InferenceComponentName=inference_component_name)
+                if inference_component_name
+                else client.invoke_endpoint(**invoke_args)
             )
         except Exception as e:
-            status_code: Final = getattr(e, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode", 500)
-            error_message: Final = getattr(e, "response", {}).get("Error", {}).get("Message", str(e))
-            raise SagemakerError(status_code=status_code, message=error_message)
+            status_code, error_message = client_error_details(e)
+            raise SagemakerError(status_code=status_code, message=with_inference_component_hint(error_message))
 
         response = json.loads(response["Body"].read().decode("utf8"))
         ## LOGGING
@@ -628,7 +648,7 @@ class SagemakerLLM(BaseAWSLLM):
 
         # Use the request_data that was already transformed above
         return provider_config.transform_embedding_response(
-            model=model,
+            model=endpoint_name,
             raw_response=mock_response,
             model_response=model_response,
             logging_obj=logging_obj,

litellm/llms/sagemaker/embedding/openai_transformation.py

@@ -0,0 +1,146 @@
+"""
+Translate from OpenAI's `/v1/embeddings` to SageMaker's `/invoke`
+
+For endpoints whose container already serves the OpenAI embeddings API, opted into per
+deployment with the `openai/` route (`sagemaker/openai/<endpoint-name>`): the request
+goes out as `{"input": [...]}` plus the caller's extra body fields, and the response is
+read as the OpenAI embeddings object (`data[].embedding`, `usage`).
+"""
+
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.llms.openai import AllEmbeddingInputValues
+
+from httpx._models import Headers, Response
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import EmbeddingResponse, Usage
+
+from ..common_utils import STR_KEYED_MAPPING, SagemakerError
+
+SUPPORTED_OPENAI_PARAMS: Final = ("dimensions", "encoding_format", "user")
+
+
+class OpenAIEmbeddingUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    prompt_tokens: int = 0
+    total_tokens: int | None = None
+
+
+class OpenAIEmbeddingRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    embedding: tuple[float, ...] | str
+    index: int | None = None
+
+
+class OpenAIEmbeddingsPayload(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    data: tuple[OpenAIEmbeddingRow, ...]
+    model: str | None = None
+    usage: OpenAIEmbeddingUsage | None = None
+
+
+class SagemakerOpenAIEmbeddingConfig(BaseEmbeddingConfig):
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: BaseConfig returns a list
+        return list(SUPPORTED_OPENAI_PARAMS)
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: callers keep adding to the mapped params
+        mapped: Final = {k: v for k, v in non_default_params.items() if k in SUPPORTED_OPENAI_PARAMS}
+        return {**optional_params, **mapped}
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str] | Headers,  # mutable-ok: SagemakerError takes the same header types
+    ) -> BaseLLMException:
+        return SagemakerError(message=error_message, status_code=status_code, headers=headers)
+
+    def transform_embedding_request(
+        self,
+        model: str,
+        input: "AllEmbeddingInputValues",
+        optional_params: Mapping[str, object],
+        headers: Mapping[str, str],
+    ) -> dict[str, object]:  # mutable-ok: BaseEmbeddingConfig returns the request body as a dict
+        extra_body: Final = optional_params.get("extra_body")
+        extra_fields: Final[Mapping[str, object]] = (
+            STR_KEYED_MAPPING.validate_python(extra_body) if isinstance(extra_body, dict) else {}
+        )
+        top_level_params: Final = {k: v for k, v in optional_params.items() if k != "extra_body"}
+        return {"input": input, **top_level_params, **extra_fields}
+
+    def transform_embedding_response(
+        self,
+        model: str,
+        raw_response: Response,
+        model_response: "EmbeddingResponse",
+        logging_obj: "LiteLLMLoggingObj",
+        api_key: str | None = None,
+        request_data: Mapping[str, object] | None = None,
+        optional_params: Mapping[str, object] | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+    ) -> "EmbeddingResponse":
+        try:
+            response_data: Final = raw_response.json()
+        except ValueError as e:
+            raise SagemakerError(
+                message=f"Failed to parse response: {e}",
+                status_code=raw_response.status_code,
+            )
+
+        try:
+            payload: Final = OpenAIEmbeddingsPayload.model_validate(response_data)
+        except ValidationError as e:
+            raise SagemakerError(
+                status_code=500,
+                message=(
+                    "Unexpected response format. Expected an OpenAI embeddings object with a 'data' list of "
+                    f"{{'embedding': [...]}} rows: {e}"
+                ),
+            )
+
+        prompt_tokens: Final = payload.usage.prompt_tokens if payload.usage else 0
+        total_tokens: Final = (
+            payload.usage.total_tokens if payload.usage and payload.usage.total_tokens is not None else prompt_tokens
+        )
+
+        return model_response.model_copy(
+            update={
+                "object": "list",
+                "data": [
+                    {
+                        "object": "embedding",
+                        "index": row.index if row.index is not None else idx,
+                        "embedding": row.embedding if isinstance(row.embedding, str) else list(row.embedding),
+                    }
+                    for idx, row in enumerate(payload.data)
+                ],
+                "model": payload.model or model,
+                "usage": Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=total_tokens),
+            }
+        )
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseConfig returns the request headers as a dict
+        return {"Content-Type": "application/json"}

litellm/llms/sagemaker/embedding/transformation.py

@@ -19,6 +19,20 @@ from litellm.types.utils import EmbeddingResponse, Usage
 
 from ..common_utils import SagemakerError
 from .cohere_transformation import SagemakerCohereEmbeddingConfig
+from .openai_transformation import SagemakerOpenAIEmbeddingConfig
+
+OPENAI_COMPATIBLE_ROUTE: Final = "openai/"
+
+
+def _model_specific_config(model: str) -> BaseEmbeddingConfig | None:
+    if model.startswith(OPENAI_COMPATIBLE_ROUTE):
+        return SagemakerOpenAIEmbeddingConfig()
+    model_lower: Final = model.lower()
+    if "voyage" in model_lower:
+        return VoyageEmbeddingConfig()
+    if "cohere" in model_lower:
+        return SagemakerCohereEmbeddingConfig()
+    return None
 
 
 class SagemakerEmbeddingConfig(BaseEmbeddingConfig):
@@ -35,25 +49,22 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig):
         Factory method to get the appropriate embedding config based on model type
 
         Args:
-            model: The model name
+            model: The model name, optionally prefixed with the `openai/` route for
+                endpoints whose container serves the OpenAI embeddings API
 
         Returns:
             Appropriate embedding config instance
         """
-        model_lower: Final = model.lower()
-        if "voyage" in model_lower:
-            return VoyageEmbeddingConfig()
-        if "cohere" in model_lower:
-            return SagemakerCohereEmbeddingConfig()
-        return cls()
+        specific_config: Final = _model_specific_config(model)
+        return cls() if specific_config is None else specific_config
+
+    @staticmethod
+    def get_endpoint_name(model: str) -> str:
+        return model.removeprefix(OPENAI_COMPATIBLE_ROUTE)
 
     def get_supported_openai_params(self, model: str) -> list[str]:
-        model_lower: Final = model.lower()
-        if "voyage" in model_lower:
-            return VoyageEmbeddingConfig().get_supported_openai_params(model)
-        if "cohere" in model_lower:
-            return SagemakerCohereEmbeddingConfig().get_supported_openai_params(model)
-        return []
+        specific_config: Final = _model_specific_config(model)
+        return [] if specific_config is None else specific_config.get_supported_openai_params(model)
 
     def map_openai_params(
         self,

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -1299,11 +1299,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             elif "AUDIO" not in optional_params["responseModalities"]:
                 optional_params["responseModalities"].append("AUDIO")
 
-        # Set default temperature to 1.0 for Gemini 3 models if not specified
-        if VertexGeminiConfig._is_gemini_3_or_newer(model):
-            if "temperature" not in optional_params:
-                optional_params["temperature"] = 1.0
-
         self._drop_search_tools_mixed_with_functions(optional_params)
 
         return optional_params
@@ -1963,20 +1958,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         tool_use_prompt_tokens: Final = usage_metadata.get("toolUsePromptTokenCount") or None
 
-        prompt_tokens_details = PromptTokensDetailsWrapper(
-            cached_tokens=cached_tokens,
-            audio_tokens=prompt_audio_tokens,
-            text_tokens=prompt_text_tokens,
-            image_tokens=prompt_image_tokens,
-            video_tokens=prompt_video_tokens,
-            tool_use_tokens=tool_use_prompt_tokens,
-        )
-
         billable_tool_use_prompt_tokens: Final = (
             0
             if VertexGeminiConfig._response_has_search_grounding(completion_response)
             else (tool_use_prompt_tokens or 0)
         )
+        billable_text_tokens: Final = (
+            prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
+        )
+
+        prompt_tokens_details = PromptTokensDetailsWrapper(
+            cached_tokens=cached_tokens,
+            audio_tokens=prompt_audio_tokens,
+            text_tokens=billable_text_tokens,
+            image_tokens=prompt_image_tokens,
+            video_tokens=prompt_video_tokens,
+            tool_use_tokens=tool_use_prompt_tokens,
+        )
 
         completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0)
         if not VertexGeminiConfig.is_candidate_token_count_inclusive(usage_metadata) and reasoning_tokens:

litellm/model_prices_and_context_window_backup.json

@@ -16022,6 +16022,26 @@
         "prompt_cache_min_tokens": 1024,
         "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     },
+    "cloudflare/clef-omni": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 64000,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
+        "supports_audio_input": true,
+        "supports_video_input": true
+    },
+    "cloudflare/@cf/cloudflare/clef-omni": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 64000,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/",
+        "supports_audio_input": true,
+        "supports_video_input": true
+    },
     "cloudflare/clef": {
         "input_cost_per_token": 2.4e-07,
         "litellm_provider": "cloudflare",
@@ -16614,6 +16634,144 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "coralbricks/glm-5.3-fast": {
+        "cache_creation_input_token_cost": 1.68e-06,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1.12e-06,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "coralbricks/glm-5.3-fp4": {
+        "cache_creation_input_token_cost": 1.68e-06,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1.12e-06,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "coralbricks/glm-5.3": {
+        "cache_creation_input_token_cost": 1.68e-06,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1.12e-06,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "coralbricks/deepseek-v4.1-flash-fast": {
+        "cache_creation_input_token_cost": 9e-08,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1e-08,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "coralbricks/deepseek-v4.1-flash-fast-fp4": {
+        "cache_creation_input_token_cost": 9e-08,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1e-08,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "coralbricks/deepseek-v4.1-flash": {
+        "cache_creation_input_token_cost": 9e-08,
+        "cache_read_input_token_cost": 0.0,
+        "input_cost_per_token": 1e-08,
+        "litellm_provider": "coralbricks",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://www.coralbricks.ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "computer-use-preview": {
         "deprecation_date": "2026-07-23",
         "input_cost_per_token": 3e-06,
@@ -16644,6 +16802,36 @@
         "supports_vision": true,
         "source": "https://platform.openai.com/docs/models/computer-use-preview"
     },
+    "computer-use-preview-2025-03-11": {
+        "deprecation_date": "2026-07-23",
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 8192,
+        "max_output_tokens": 1024,
+        "max_tokens": 1024,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "source": "https://developers.openai.com/api/docs/models/computer-use-preview"
+    },
     "deepseek-chat": {
         "cache_read_input_token_cost": 2.8e-08,
         "deprecation_date": "2026-07-24",
@@ -33449,6 +33637,44 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "gpt-audio-mini-2025-10-06": {
+        "deprecation_date": "2026-07-23",
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "source": "https://developers.openai.com/api/docs/models/gpt-audio-mini",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
     "gpt-audio-mini-2025-12-15": {
         "input_cost_per_audio_token": 1e-05,
         "input_cost_per_token": 6e-07,
@@ -33609,6 +33835,34 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gpt-4o-mini-search-preview-2025-03-11": {
+        "deprecation_date": "2026-07-23",
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 1.5e-07,
+        "input_cost_per_token_batches": 7.5e-08,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "output_cost_per_token_batches": 3e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.025,
+            "search_context_size_low": 0.025,
+            "search_context_size_medium": 0.025
+        },
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "source": "https://developers.openai.com/api/docs/models/gpt-4o-mini-search-preview"
+    },
     "gpt-4o-mini-transcribe": {
         "input_cost_per_audio_token": 1.25e-06,
         "input_cost_per_token": 1.25e-06,
@@ -33689,6 +33943,34 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gpt-4o-search-preview-2025-03-11": {
+        "deprecation_date": "2026-07-23",
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_batches": 1.25e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_token_batches": 5e-06,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.025,
+            "search_context_size_low": 0.025,
+            "search_context_size_medium": 0.025
+        },
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "source": "https://developers.openai.com/api/docs/models/gpt-4o-search-preview"
+    },
     "gpt-4o-transcribe": {
         "input_cost_per_audio_token": 2.5e-06,
         "input_cost_per_token": 2.5e-06,
@@ -34942,13 +35224,17 @@
     },
     "gpt-6-luna": {
         "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_decisions": 1e-07,
         "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
         "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
         "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
         "cache_creation_input_token_cost_flex": 6.25e-08,
         "cache_creation_input_token_cost_priority": 2.5e-07,
         "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_decisions": 1e-07,
         "cache_read_input_token_cost_above_272k_tokens": 2e-08,
+        "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
         "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
         "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
         "cache_read_input_token_cost_batches": 5e-09,
@@ -34958,7 +35244,9 @@
         "cache_read_input_token_cost_flex": 5e-09,
         "cache_read_input_token_cost_priority": 2e-08,
         "input_cost_per_token": 1e-07,
+        "input_cost_per_token_decisions": 1e-07,
         "input_cost_per_token_above_272k_tokens": 2e-07,
+        "input_cost_per_token_above_272k_tokens_decisions": 2e-07,
         "input_cost_per_token_above_272k_tokens_flex": 1e-07,
         "input_cost_per_token_above_272k_tokens_priority": 4e-07,
         "input_cost_per_token_batches": 5e-08,
@@ -34971,7 +35259,9 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 5e-07,
+        "output_cost_per_token_decisions": 0.0,
         "output_cost_per_token_above_272k_tokens": 7.5e-07,
+        "output_cost_per_token_above_272k_tokens_decisions": 0.0,
         "output_cost_per_token_above_272k_tokens_flex": 3.75e-07,
         "output_cost_per_token_above_272k_tokens_priority": 1.5e-06,
         "output_cost_per_token_batches": 2.5e-07,
@@ -38843,6 +39133,49 @@
         ],
         "supports_audio_input": true
     },
+    "meta/muse-wonder-1.0": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "meta",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.0025,
+            "search_context_size_low": 0.0025,
+            "search_context_size_medium": 0.0025
+        },
+        "source": "https://dev.meta.ai/docs/pricing-rate-limits",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "meta_llama/Llama-3.3-70B-Instruct": {
         "litellm_provider": "meta_llama",
         "max_input_tokens": 128000,
@@ -41481,6 +41814,38 @@
         "supports_pdf_input": true,
         "deprecation_date": "2026-07-23"
     },
+    "o3-deep-research-2025-06-26": {
+        "deprecation_date": "2026-07-23",
+        "cache_read_input_token_cost": 2.5e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 100000,
+        "max_tokens": 100000,
+        "mode": "responses",
+        "output_cost_per_token": 4e-05,
+        "source": "https://developers.openai.com/api/docs/models/o3-deep-research",
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": false,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_tool_choice": false,
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "o3-2025-04-16": {
         "cache_read_input_token_cost": 5e-07,
         "cache_read_input_token_cost_flex": 2.5e-07,
@@ -41711,6 +42076,38 @@
         "supports_pdf_input": true,
         "deprecation_date": "2026-07-23"
     },
+    "o4-mini-deep-research-2025-06-26": {
+        "deprecation_date": "2026-07-23",
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 100000,
+        "max_tokens": 100000,
+        "mode": "responses",
+        "output_cost_per_token": 8e-06,
+        "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research",
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": false,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_tool_choice": false,
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "o4-mini-2025-04-16": {
         "cache_read_input_token_cost": 2.75e-07,
         "cache_read_input_token_cost_flex": 1.38e-07,
@@ -43006,13 +43403,21 @@
         "output_cost_per_token": 2.8e-07
     },
     "perplexity/pplx-decider-v1-27b": {
-        "input_cost_per_token": 4e-08,
+        "input_cost_per_token": 2e-08,
         "litellm_provider": "perplexity",
         "max_input_tokens": 262144,
         "mode": "evaluation",
         "output_cost_per_token": 0.0,
         "source": "https://docs.perplexity.ai/api-reference/decisions-post"
     },
+    "perplexity/pplx-decider-v1.1-27b": {
+        "input_cost_per_token": 2e-08,
+        "litellm_provider": "perplexity",
+        "max_input_tokens": 262144,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.perplexity.ai/docs/getting-started/pricing"
+    },
     "perplexity/sonar": {
         "input_cost_per_token": 1e-06,
         "litellm_provider": "perplexity",
@@ -51404,6 +51809,70 @@
         "litellm_provider": "vertex_ai",
         "mode": "vector_store"
     },
+    "vertex_ai/semantic-ranker-default": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-default-002": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-default-003": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-default-004": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-default@latest": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-fast": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-fast-004": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
+    "vertex_ai/semantic-ranker-fast@latest": {
+        "input_cost_per_query": 0.001,
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "vertex_ai",
+        "mode": "rerank",
+        "output_cost_per_token": 0.0,
+        "source": "https://cloud.google.com/generative-ai-app-builder/pricing"
+    },
     "openai/container": {
         "code_interpreter_cost_per_session": 0.03,
         "litellm_provider": "openai",
@@ -65714,7 +66183,6 @@
         "source": "https://api.together.xyz/v1/models"
     },
     "together_ai/together/Tev1-4B-experimental": {
-        "cache_read_input_token_cost": 4.2e-08,
         "input_cost_per_token": 4.2e-08,
         "litellm_provider": "together_ai",
         "max_input_tokens": 32768,
@@ -69135,6 +69603,15 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "fireworks_ai/accounts/fireworks/routers/rune": {
+        "cache_read_input_token_cost": 8.4e-09,
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "fireworks_ai",
+        "max_input_tokens": 262144,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.fireworks.ai"
+    },
     "fireworks_ai/accounts/fireworks/models/ember-1": {
         "cache_read_input_token_cost": 3e-07,
         "cache_read_input_token_cost_priority": 3.75e-07,
@@ -71966,5 +72443,97 @@
         "supports_system_messages": false,
         "supports_vision": false,
         "supports_function_calling": false
+    },
+    "vertex_ai/gemini-robotics-er-2-preview": {
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_batches": 5e-08,
+        "input_cost_per_audio_token": 1e-06,
+        "input_cost_per_token": 1e-06,
+        "input_cost_per_token_batches": 5e-07,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 5e-06,
+        "output_cost_per_token": 5e-06,
+        "output_cost_per_token_batches": 2.5e-06,
+        "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "web_search_billing_unit": "per_query"
+    },
+    "gemini-robotics-er-2-preview": {
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_batches": 5e-08,
+        "input_cost_per_audio_token": 1e-06,
+        "input_cost_per_token": 1e-06,
+        "input_cost_per_token_batches": 5e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 5e-06,
+        "output_cost_per_token": 5e-06,
+        "output_cost_per_token_batches": 2.5e-06,
+        "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "web_search_billing_unit": "per_query"
     }
 }

litellm/provider_endpoints_support_backup.json

@@ -618,6 +618,24 @@
         "interactions": true
       }
     },
+    "coralbricks": {
+      "display_name": "CoralBricks (`coralbricks`)",
+      "url": "https://docs.litellm.ai/docs/providers/coralbricks",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false,
+        "interactions": false
+      }
+    },
     "cortecs": {
       "display_name": "Cortecs (`cortecs`)",
       "url": "https://docs.litellm.ai/docs/providers/cortecs",

litellm/proxy/_experimental/mcp_server/db.py

@@ -1790,7 +1790,7 @@ async def store_user_oauth_credential(
         payload["refresh_token"] = refresh_token
     if expires_at:
         payload["expires_at"] = expires_at
-    if scopes:
+    if scopes is not None:
         payload["scopes"] = scopes
 
     # Guard against silently overwriting a BYOK credential with an OAuth token.
@@ -2123,9 +2123,7 @@ async def refresh_user_oauth_token(
     new_refresh_token: Final[str | None] = body.get("refresh_token") or refresh_token
 
     raw_scope: Final = body.get("scope")
-    scopes: list[str] | None = (raw_scope.split() if isinstance(raw_scope, str) and raw_scope else None) or cred.get(
-        "scopes"
-    )
+    scopes: Final = raw_scope.split() if isinstance(raw_scope, str) else cred.get("scopes")
 
     await store_user_oauth_credential(
         prisma_client=prisma_client,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -12,6 +12,8 @@ from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 import httpx
 from fastapi import APIRouter, Depends, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
+from mcp.client.auth.utils import get_client_metadata_scopes, union_scopes
+from mcp.shared.auth import OAuthMetadata
 from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
@@ -110,6 +112,7 @@ from litellm.types.mcp import MCPAuth, MCPCredentials
 from litellm.types.mcp_server.mcp_server_manager import MCPServer, MCPTokenEndpointAuthMethod
 
 if TYPE_CHECKING:
+    from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload
     from litellm.proxy._types import LiteLLM_MCPServerTable
 
 # TTL cache for upstream OAuth metadata fetched from pass-through MCP servers.
@@ -219,6 +222,8 @@ def encode_state_with_base_url(
     expected_issuer: str | None = None,
     authorization_response_iss_parameter_supported: bool = False,
     oauth_nonce: str | None = None,
+    managed_scope: str | None = None,
+    scope_user_id: str | None = None,
 ) -> str:
     """
     Encode the base_url, original state, and PKCE parameters using encryption.
@@ -251,6 +256,7 @@ def encode_state_with_base_url(
     """
     state_data: Final = {
         "oauth_nonce": oauth_nonce,
+        **({"managed_scope": managed_scope, "scope_user_id": scope_user_id} if managed_scope is not None else {}),
         "base_url": base_url,
         "original_state": original_state,
         "code_challenge": code_challenge,
@@ -290,6 +296,18 @@ def decode_state_hash(encrypted_state: str) -> dict:
     return state_data
 
 
+_MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
+
+
+class _ManagedAuthorizationCode(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    upstream_code: str = Field(min_length=1)
+    mcp_server_id: str = Field(min_length=1)
+    issuer: str | None = None
+    requested_scope: str
+    scope_user_id: str | None = None
+
+
 _BRIDGE_AUTH_CODE_PREFIX: Final = "llm_bcode_"
 
 
@@ -298,6 +316,8 @@ class _BridgeAuthorizationCode(LiteLLMBaseModel):
 
     model_config = ConfigDict(frozen=True)
     oauth_nonce: str | None = None
+    requested_scope: str | None = None
+    expected_issuer: str | None = None
     upstream_code: str = Field(min_length=1)
     litellm_user_id: str = Field(min_length=1)
     mcp_server_id: str = Field(min_length=1)
@@ -314,6 +334,8 @@ def seal_bridge_authorization_code(
     litellm_user_id: str,
     mcp_server_id: str,
     oauth_nonce: str | None = None,
+    requested_scope: str | None = None,
+    expected_issuer: str | None = None,
 ) -> str:
     """Seal the upstream authorization code and the SSO-captured litellm user into a gateway
     authorization code. The DCR client only echoes this opaque value back at the token endpoint; the
@@ -328,6 +350,8 @@ def seal_bridge_authorization_code(
             "litellm_user_id": litellm_user_id,
             "mcp_server_id": mcp_server_id,
             "oauth_nonce": oauth_nonce,
+            "requested_scope": requested_scope,
+            "expected_issuer": expected_issuer,
         },
         sort_keys=True,
     )
@@ -643,6 +667,7 @@ async def _store_per_user_token_server_side(
     token_response: dict[str, Any],
     identity_binding_proof: str | None = None,
     cimd_client_id: str | None = None,
+    requested_scope: str | None = None,
 ) -> None:
     """Persist the OAuth token server-side and warm the Redis cache.
 
@@ -667,8 +692,8 @@ async def _store_per_user_token_server_side(
         expires_in = None
 
     refresh_token: Final[str | None] = token_response.get("refresh_token") or None
-    raw_scope: Final = token_response.get("scope")
-    scopes: Final[list | None] = raw_scope.split() if isinstance(raw_scope, str) and raw_scope else None
+    raw_scope: Final[object] = token_response.get("scope", requested_scope)
+    scopes: Final[list[str] | None] = raw_scope.split() if isinstance(raw_scope, str) else None
 
     try:
         prisma_client: Final = get_prisma_client_or_throw("Database not connected. Cannot store per-user OAuth token.")
@@ -954,6 +979,46 @@ async def _resolve_oauth_authorization_user(
     return user_id
 
 
+async def _managed_authorization_scope(
+    request: Request, server: MCPServer, scope: str | None, authorized_user_id: str | None
+) -> tuple[str | None, str | None]:
+    from litellm.proxy._experimental.mcp_server.db import get_user_oauth_credential
+    from litellm.proxy.proxy_server import prisma_client
+
+    gateway_credential: Final = authorized_user_id is None and await oauth_authorization_uses_gateway_credential(
+        request
+    )
+    session_user: Final = (
+        _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None
+    )
+    scope_user_id: Final = authorized_user_id or (
+        await authorize_oauth_credential_request(request, server.server_id)
+        if gateway_credential
+        else session_user
+        if session_user and await _user_can_reach_mcp_server(session_user, server.server_id)
+        else None
+    )
+    prior_grant: Final = (
+        await get_user_oauth_credential(prisma_client, scope_user_id, server.server_id)
+        if scope_user_id and prisma_client is not None
+        else None
+    )
+    merged_scope: Final = union_scopes(" ".join(prior_grant.get("scopes") or ()) if prior_grant else None, scope)
+    selected_scope: Final = (
+        get_client_metadata_scopes(
+            merged_scope,
+            None,
+            OAuthMetadata.model_validate(server.authorization_server_metadata)
+            if server.authorization_server_metadata is not None
+            else None,
+            ["authorization_code", "refresh_token"],
+        )
+        if merged_scope is not None
+        else None
+    )
+    return selected_scope, scope_user_id
+
+
 async def authorize_with_server(
     request: Request,
     mcp_server: MCPServer,
@@ -1029,6 +1094,17 @@ async def authorize_with_server(
             return subject
         litellm_user_id = subject
 
+    managed_per_user: Final = resolved_server.is_gateway_managed_oauth2 and resolved_server.needs_user_oauth_token
+    initial_scope: Final = union_scopes(
+        "openid" if enforce_binding else None,
+        scope or (" ".join(resolved_server.scopes) if resolved_server.scopes else None),
+    )
+    selected_scope, scope_user_id = (
+        await _managed_authorization_scope(request, resolved_server, initial_scope, litellm_user_id)
+        if managed_per_user
+        else (initial_scope, None)
+    )
+
     oauth_nonce: Final = secrets.token_urlsafe(32) if enforce_binding else None
     encoded_state: Final = encode_state_with_base_url(
         base_url=base_url,
@@ -1038,7 +1114,11 @@ async def authorize_with_server(
         code_challenge_method=code_challenge_method,
         client_redirect_uri=redirect_uri,
         litellm_user_id=litellm_user_id,
-        mcp_server_id=resolved_server.server_id if (litellm_user_id or ephemeral_dcr_client) else None,
+        mcp_server_id=resolved_server.server_id
+        if (litellm_user_id or ephemeral_dcr_client or managed_per_user)
+        else None,
+        managed_scope=selected_scope if managed_per_user else None,
+        scope_user_id=scope_user_id,
         dcr_client_id=ephemeral_dcr_client.client_id if ephemeral_dcr_client else None,
         dcr_client_secret=ephemeral_dcr_client.client_secret if ephemeral_dcr_client else None,
         dcr_token_endpoint_auth_method=ephemeral_dcr_client.token_endpoint_auth_method
@@ -1057,13 +1137,8 @@ async def authorize_with_server(
     }
     if oauth_nonce:
         params["nonce"] = oauth_nonce
-    if scope:
-        params["scope"] = scope
-    elif resolved_server.scopes:
-        params["scope"] = " ".join(resolved_server.scopes)
-
-    if enforce_binding and "openid" not in params.get("scope", "").split():
-        params["scope"] = f"openid {params.get('scope', '')}".strip()
+    if selected_scope:
+        params["scope"] = selected_scope
 
     if code_challenge:
         params["code_challenge"] = code_challenge
@@ -1090,18 +1165,11 @@ def _token_credential_source(mcp_server: MCPServer) -> CredentialSource:
     return "gateway_stored" if mcp_server.client_id or get_cimd_client_id(mcp_server) else "caller_supplied"
 
 
-async def _saved_cimd_refresh_client_id(
+async def _saved_refresh_credential(
     server: MCPServer, user_id: str | None, refresh_token: str | None
-) -> str | None:
-    """Reuse only the client identity bound to this caller's presented refresh grant."""
-    if (
-        not user_id
-        or not refresh_token
-        or not server.needs_user_oauth_token
-        or server.auth_type != MCPAuth.oauth2
-        or server.client_id
-        or server.client_secret
-    ):
+) -> "OAuthCredentialPayload | None":
+    """Reuse only metadata bound to this caller's presented refresh grant."""
+    if not user_id or not refresh_token or not server.needs_user_oauth_token or server.auth_type != MCPAuth.oauth2:
         return None
     from litellm.proxy import proxy_server
     from litellm.proxy._experimental.mcp_server.db import get_user_oauth_credential
@@ -1117,7 +1185,33 @@ async def _saved_cimd_refresh_client_id(
     stored_refresh: Final = credential.get("refresh_token")
     if not stored_refresh or not secrets.compare_digest(stored_refresh.encode(), refresh_token.encode()):
         return None
-    return credential.get("cimd_client_id")
+    return credential
+
+
+def _open_managed_authorization_code(
+    code: str | None, server: MCPServer, grant_type: str, user_id: str | None
+) -> _ManagedAuthorizationCode | None:
+    if not isinstance(code, str) or not code.startswith(_MANAGED_AUTH_CODE_PREFIX):
+        return None
+    decoded_code: Final = decrypt_value_helper(
+        code[len(_MANAGED_AUTH_CODE_PREFIX) :], "managed_authorization_code", return_original_value=False
+    )
+    if not isinstance(decoded_code, str):
+        raise HTTPException(status_code=400, detail="Invalid authorization code")
+    try:
+        managed_code: Final = _ManagedAuthorizationCode.model_validate_json(decoded_code)
+    except ValidationError as exc:
+        raise HTTPException(status_code=400, detail="Invalid authorization code") from exc
+    if (
+        grant_type != "authorization_code"
+        or not server.is_gateway_managed_oauth2
+        or not server.needs_user_oauth_token
+        or managed_code.mcp_server_id != server.server_id
+        or managed_code.issuer != server.issuer
+        or (managed_code.scope_user_id is not None and managed_code.scope_user_id != user_id)
+    ):
+        raise HTTPException(status_code=400, detail="Authorization code does not match this OAuth flow")
+    return managed_code
 
 
 async def exchange_token_with_server(
@@ -1159,10 +1253,18 @@ async def exchange_token_with_server(
         if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None
         else None
     )
-    cimd_client_id: Final = (
-        await _saved_cimd_refresh_client_id(resolved_server, request_user_id, refresh_token)
+    managed_code: Final = _open_managed_authorization_code(code, resolved_server, grant_type, request_user_id)
+    authorization_code: Final = managed_code.upstream_code if managed_code else code
+
+    refresh_credential: Final = (
+        await _saved_refresh_credential(resolved_server, request_user_id, refresh_token)
         if grant_type == "refresh_token"
         else None
+    )
+    cimd_client_id: Final = (
+        refresh_credential.get("cimd_client_id")
+        if refresh_credential and not resolved_server.client_id and not resolved_server.client_secret
+        else None
     ) or get_cimd_client_id(resolved_server)
 
     # The id, secret, and token-endpoint auth method must come from the same source. When the
@@ -1234,7 +1336,7 @@ async def exchange_token_with_server(
         )
     else:
         refresh_ownership = None  # rebind-ok: grant-specific branches assign one ownership value
-        if not code:
+        if not authorization_code:
             raise HTTPException(
                 status_code=400,
                 detail="code is required for authorization_code grant",
@@ -1244,14 +1346,18 @@ async def exchange_token_with_server(
         # below uses the upstream code, and the mint binds the envelope to the recovered user. Bind the
         # sealed server to this request so a code minted for one bridge server cannot be spent at another.
         # A raw upstream code (scripted path) opens to None and the code is used as-is.
-        bridge_identity = open_bridge_authorization_code(code)
-        if bridge_identity is not None:
-            if bridge_identity.mcp_server_id != resolved_server.server_id:
-                raise HTTPException(
-                    status_code=400,
-                    detail="Authorization code was issued for a different MCP server",
-                )
-            code = bridge_identity.upstream_code
+        bridge_identity = open_bridge_authorization_code(authorization_code)
+        if bridge_identity is not None and (
+            bridge_identity.mcp_server_id != resolved_server.server_id
+            or (
+                bridge_identity.requested_scope is not None
+                and bridge_identity.expected_issuer != resolved_server.issuer
+            )
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail="Authorization code was issued for a different MCP server",
+            )
         binding: Final = resolved_server.oauth_identity_binding
         if binding is not None and binding.mode == "enforce":
             if bridge_identity is None or not bridge_identity.oauth_nonce:
@@ -1273,7 +1379,7 @@ async def exchange_token_with_server(
         resolved_redirect_uri: Final = redirect_uri if bridge_token_relay else f"{proxy_base_url}/callback"
         token_data = {
             "grant_type": "authorization_code",
-            "code": code,
+            "code": bridge_identity.upstream_code if bridge_identity else authorization_code,
             "redirect_uri": resolved_redirect_uri,
             **token_request.body,
         }
@@ -1353,6 +1459,16 @@ async def exchange_token_with_server(
         else None
     )
 
+    requested_scope: Final = (
+        managed_code.requested_scope
+        if managed_code is not None
+        else bridge_identity.requested_scope
+        if bridge_identity is not None
+        else scope or " ".join(refresh_credential.get("scopes") or ())
+        if refresh_credential
+        else None
+    )
+
     # Store server-side when the server is configured for per-user OAuth and
     # the calling client has provided a valid LiteLLM identity.
     # Errors are non-fatal: the token is still returned to the client.
@@ -1379,6 +1495,7 @@ async def exchange_token_with_server(
                         user_id=user_id,
                         token_response=token_response,
                         identity_binding_proof=binding_proof,
+                        requested_scope=requested_scope,
                         **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}),
                     )
                 else:
@@ -1431,8 +1548,10 @@ async def exchange_token_with_server(
         result["expires_in"] = token_response["expires_in"]
     if token_response.get("refresh_token"):
         result["refresh_token"] = token_response["refresh_token"]
-    if token_response.get("scope"):
-        result["scope"] = token_response["scope"]
+    token_fields: Final = TypeAdapter[dict[str, object]](dict[str, object]).validate_python(token_response)
+    effective_scope: Final = token_fields.get("scope", requested_scope)
+    if effective_scope is not None:
+        result["scope"] = effective_scope
 
     # RFC 6749 §5.1: token responses must not be cached.
     return JSONResponse(result, headers=TOKEN_NO_CACHE_HEADERS)
@@ -2116,6 +2235,7 @@ async def authorize_mcp_session(
     code_challenge_method: str | None = None,
     response_type: str | None = None,
     resource: str | None = None,
+    scope: str | None = None,
 ) -> Response:
     return aggregate_authorize(
         request=request,
@@ -2127,6 +2247,7 @@ async def authorize_mcp_session(
         response_type=response_type,
         session_user_id=_session_cookie_user_id(request),
         resource=resource,
+        scope=scope,
     )
 
 
@@ -2169,6 +2290,7 @@ async def authorize(
             response_type=response_type,
             session_user_id=_session_cookie_user_id(request),
             resource=resource,
+            scope=scope,
         )
 
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
@@ -2281,7 +2403,9 @@ async def token_endpoint(
     )
 
 
-async def _vendor_credential_state(user_id: str, server_id: str) -> VendorCredentialState:
+async def _vendor_credential_state(
+    user_id: str, server_id: str, requested_scopes: tuple[str, ...] = ()
+) -> VendorCredentialState:
     """Whether the gateway itself can see a live vendor credential for this user and server.
 
     The one reading of "authorized" the connect page displays and the finish step enforces, so
@@ -2299,7 +2423,10 @@ async def _vendor_credential_state(user_id: str, server_id: str) -> VendorCreden
         credential: Final = await get_user_oauth_credential(prisma_client, user_id, server_id)
     except Exception:  # noqa: BLE001  # a credential-read fault must fail the scoped grant closed
         return "unavailable"
-    return "absent" if oauth_grant_state(credential) == "absent" else "present"
+    if oauth_grant_state(credential) == "absent":
+        return "absent"
+    granted_scopes: Final = credential.get("scopes") if credential else None
+    return "present" if set(requested_scopes).issubset(granted_scopes or ()) else "absent"
 
 
 @router.get("/authorize/flow")
@@ -2542,6 +2669,11 @@ async def callback(
         mcp_server_id: Final = state_data.get("mcp_server_id")
         dcr_client_id: Final = state_data.get("dcr_client_id")
         dcr_client_secret: Final = state_data.get("dcr_client_secret")
+        managed_scope: Final = issuer_state.get("managed_scope")
+        expected_issuer: Final = TypeAdapter[str | None](str | None).validate_python(
+            issuer_state.get("expected_issuer")
+        )
+        scope_owner: Final = TypeAdapter[str | None](str | None).validate_python(issuer_state.get("scope_user_id"))
         forwarded_code = code
         if isinstance(litellm_user_id, str) and litellm_user_id and isinstance(mcp_server_id, str) and mcp_server_id:
             forwarded_code = seal_bridge_authorization_code(
@@ -2549,6 +2681,8 @@ async def callback(
                 litellm_user_id=litellm_user_id,
                 mcp_server_id=mcp_server_id,
                 oauth_nonce=state_data.get("oauth_nonce"),
+                requested_scope=managed_scope if isinstance(managed_scope, str) else None,
+                expected_issuer=expected_issuer,
             )
         elif isinstance(dcr_client_id, str) and dcr_client_id and isinstance(mcp_server_id, str) and mcp_server_id:
             forwarded_code = seal_passthrough_authorization_code(
@@ -2561,7 +2695,23 @@ async def callback(
                 ),
             )
 
-        params = {"code": forwarded_code, "state": original_state}
+        managed_code: Final = (
+            _ManagedAuthorizationCode(
+                upstream_code=forwarded_code,
+                mcp_server_id=mcp_server_id,
+                issuer=expected_issuer,
+                requested_scope=managed_scope,
+                scope_user_id=scope_owner,
+            )
+            if isinstance(managed_scope, str) and isinstance(mcp_server_id, str) and not litellm_user_id
+            else None
+        )
+        returned_code: Final = (
+            _MANAGED_AUTH_CODE_PREFIX + encrypt_value_helper(managed_code.model_dump_json())
+            if managed_code is not None
+            else forwarded_code
+        )
+        params = {"code": returned_code, "state": original_state}
         complete_returned_url = _append_query_params(redirect_uri, params)
         response = RedirectResponse(url=complete_returned_url, status_code=302)
         _clear_oauth_state_cookie(response, request, state)
@@ -2807,11 +2957,16 @@ async def _build_oauth_protected_resource_response(
     if obo_response is not None:
         return obo_response
 
+    resource_scopes: Final = [
+        value
+        for value in (mcp_server.scopes or () if mcp_server else ())
+        if value != "offline_access" or not mcp_server or not mcp_server.is_gateway_managed_oauth2
+    ]
     if mcp_server is not None and mcp_server.advertises_gateway_authorization_server:
         return {
             "authorization_servers": [f"{request_base_url}/mcp"],
             "resource": resource_url,
-            "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []),
+            "scopes_supported": resource_scopes,
         }
 
     if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange:
@@ -2822,7 +2977,7 @@ async def _build_oauth_protected_resource_response(
             (f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}")
         ],
         "resource": resource_url,
-        "scopes_supported": (mcp_server.scopes if mcp_server and mcp_server.scopes else []),
+        "scopes_supported": resource_scopes,
     }
 
 

litellm/proxy/_experimental/mcp_server/exceptions.py

@@ -4,6 +4,8 @@ from typing import Final
 
 from fastapi import HTTPException
 
+from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope
+
 
 class MCPServerURLCredentialsError(HTTPException):
     """A fixed, sanitized URL-credential migration error safe for operator previews."""
@@ -37,10 +39,14 @@ class MCPUpstreamAuthError(Exception):
         status_code: int,
         www_authenticate: str | None,
         server_name: str,
+        required_scope: str | None = None,
+        server_id: str | None = None,
     ) -> None:
         self.status_code = status_code
         self.www_authenticate = www_authenticate
         self.server_name = server_name
+        self.required_scope = required_scope
+        self.server_id = server_id
         super().__init__(f"Upstream MCP server {server_name!r} returned {status_code}")
 
     def to_http_exception(
@@ -73,19 +79,39 @@ class MCPUpstreamAuthError(Exception):
         the client originally targeted, matching the path-aware behaviour of
         ``get_passthrough_resource_metadata_url`` in ``oauth_utils.py``.
         """
+        aggregate_step_up: Final = (
+            self.required_scope is not None
+            and request_path is not None
+            and (request_path.rstrip("/") == "/mcp" or request_path.startswith("/toolset/"))
+        )
+        requested_scope: Final = (
+            " ".join(
+                encode_upstream_scope(self.server_id or self.server_name, item) for item in self.required_scope.split()
+            )
+            if aggregate_step_up and self.required_scope
+            else self.required_scope
+        )
         challenge: str | None = self.www_authenticate
-        if challenge is None and self.status_code == 401 and base_url:
+        if challenge is None and (self.status_code == 401 or self.required_scope is not None) and base_url:
             prefix: Final = base_url.rstrip("/")
-            if request_path and request_path.startswith(f"/{self.server_name}/mcp"):
+            if aggregate_step_up:
+                resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource"
+            elif request_path and request_path.startswith(f"/{self.server_name}/mcp"):
                 resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource/{self.server_name}/mcp"
             else:
                 resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource/mcp/{self.server_name}"
             challenge = f'Bearer resource_metadata="{resource_metadata_url}"'
+        scoped_challenge: Final = (
+            (f'{challenge}, error="insufficient_scope"' if challenge else 'Bearer error="insufficient_scope"')
+            + (f', scope="{requested_scope}"' if requested_scope else "")
+            if self.required_scope is not None
+            else challenge
+        )
         detail: Final = "Forbidden" if self.status_code == 403 else "Unauthorized"
         return HTTPException(
             status_code=self.status_code,
             detail=detail,
-            headers={"www-authenticate": challenge} if challenge else None,
+            headers={"www-authenticate": scoped_challenge} if scoped_challenge else None,
         )
 
 

litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py

@@ -10,13 +10,15 @@ becomes an outcome, never a second failure.
 
 from __future__ import annotations
 
+import re
 from collections.abc import Iterator
 from typing import Final, Literal, NamedTuple, NoReturn, TypeAlias
+from urllib.request import parse_http_list, parse_keqv_list
 
 import httpx
 import httpx2
 from mcp.types import Tool as MCPTool
-from pydantic import ConfigDict
+from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import assert_never
 
 from litellm.proxy._experimental.mcp_server.exceptions import (
@@ -98,6 +100,29 @@ def upstream_auth_challenge(exc: BaseException) -> tuple[int, str | None] | None
     return None
 
 
+def upstream_insufficient_scope(exc: BaseException) -> str | None:
+    auth: Final = upstream_auth_challenge(exc)
+    if auth is None or auth[0] != 403 or not auth[1]:
+        return None
+    scheme, _, parameters = auth[1].partition(" ")
+    if scheme.lower() != "bearer":
+        return None
+    fields: Final = parse_http_list(parameters)
+    if any("=" not in field or not field.split("=", 1)[1] for field in fields):
+        return None
+    keys: Final = tuple(field.split("=", 1)[0].strip().lower() for field in fields)
+    if len(set(keys)) != len(keys) or any(re.fullmatch(r"[!#$%&'*+.^_`|~0-9a-z-]+", key) is None for key in keys):
+        return None
+    parsed: Final = TypeAdapter(dict[str, str]).validate_python(parse_keqv_list(fields))
+    values: Final = {key: value for key, value in zip(keys, parsed.values())}
+    if values.get("error") != "insufficient_scope":
+        return None
+    scope: Final = values.get("scope", "")
+    if scope and re.fullmatch(r"[!#-\[\]-~]+(?: [!#-\[\]-~]+)*", scope) is None:
+        return None
+    return scope
+
+
 def raise_classified_list_failure(
     exc: BaseException,
     server_name: str,

litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py

@@ -59,8 +59,11 @@ from litellm.caching.caching import DualCache
 from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     TOKEN_NO_CACHE_HEADERS,
+    UPSTREAM_SCOPE_PREFIX,
+    UpstreamScope,
     canonical_resource_uri,
     canonicalize_url_identity,
+    decode_upstream_scope,
     get_request_base_url,
     is_loopback_redirect_host,
     validate_redirect_uri_shape,
@@ -250,7 +253,9 @@ class ConsentTeam(LiteLLMBaseModel):
 class LookupVendorCredential(Protocol):
     """Injected read of a user's vendor credential for one server."""
 
-    def __call__(self, user_id: str, server_id: str, /) -> Awaitable[VendorCredentialState]: ...
+    def __call__(
+        self, user_id: str, server_id: str, requested_scopes: tuple[str, ...] = (), /
+    ) -> Awaitable[VendorCredentialState]: ...
 
 
 class LookupServerReachability(Protocol):
@@ -273,7 +278,9 @@ async def _refuse_subject_token(subject_token: str, request: Request) -> Subject
     )
 
 
-async def _unavailable_vendor_credential(user_id: str, server_id: str) -> VendorCredentialState:
+async def _unavailable_vendor_credential(
+    user_id: str, server_id: str, requested_scopes: tuple[str, ...] = ()
+) -> VendorCredentialState:
     return "unavailable"
 
 
@@ -310,6 +317,8 @@ class _ConnectFlow(LiteLLMBaseModel):
     exp: int
     resource_server_id: str | None = None
     audience: SessionAudience | None = None
+    requested_scopes: tuple[str, ...] = ()
+    upstream_scopes: tuple[UpstreamScope, ...] = ()
 
 
 class _GatewayAuthCode(LiteLLMBaseModel):
@@ -555,6 +564,7 @@ def aggregate_authorize(
     response_type: str | None,
     session_user_id: str | None,
     resource: str | None = None,
+    scope: str | None = None,
 ) -> Response:
     """The aggregate authorize verb: validate the client, require S256 PKCE, interpose
     LiteLLM sign-in, and hand the browser to the connect page with the flow sealed into a
@@ -580,6 +590,13 @@ def aggregate_authorize(
     if session_user_id is None:
         return _login_redirect(base_url, request)
     scoped_server: Final = resolve_scoped_resource_server(request, resource)
+    scope_tokens: Final = tuple(token for token in (scope or "").split() if token.startswith(UPSTREAM_SCOPE_PREFIX))
+    decoded_scopes: Final = tuple(decode_upstream_scope(token) for token in scope_tokens)
+    if any(item is None for item in decoded_scopes):
+        return _oauth_error(400, "invalid_scope", "invalid upstream permission scope")
+    upstream_scopes: Final = tuple(item for item in decoded_scopes if item is not None)
+    if scoped_server is not None and any(item.server_id != scoped_server.server_id for item in upstream_scopes):
+        return _oauth_error(400, "invalid_scope", "upstream permission does not match the requested resource")
     handle: Final = secrets.token_urlsafe(24)
     flow: Final = _new_connect_flow(
         session_user_id=session_user_id,
@@ -589,6 +606,8 @@ def aggregate_authorize(
         code_challenge=code_challenge or "",
         resource_server_id=scoped_server.server_id if scoped_server is not None else None,
         audience=None,
+        requested_scopes=tuple((scope or "").split()) if scoped_server is not None and not upstream_scopes else (),
+        upstream_scopes=upstream_scopes,
     )
     connect_url: Final = _append_query_params(f"{base_url}/ui/connect", (("connect_flow", handle),))
     response: Final = RedirectResponse(connect_url, status_code=303)
@@ -748,6 +767,8 @@ def _new_connect_flow(
     code_challenge: str,
     resource_server_id: str | None,
     audience: SessionAudience | None,
+    requested_scopes: tuple[str, ...] = (),
+    upstream_scopes: tuple[UpstreamScope, ...] = (),
 ) -> _ConnectFlow:
     now: Final = datetime.now(timezone.utc)
     return _ConnectFlow(
@@ -760,6 +781,8 @@ def _new_connect_flow(
         exp=int(now.timestamp()) + CONNECT_FLOW_TTL_SECONDS,
         resource_server_id=resource_server_id,
         audience=audience,
+        requested_scopes=requested_scopes,
+        upstream_scopes=upstream_scopes,
     )
 
 
@@ -815,20 +838,20 @@ def _open_flow_for(
 
 @catalog_operation(global_manager)
 async def _flow_target(
-    flow: _ConnectFlow, lookup_server_reachability: LookupServerReachability
+    user_id: str, server_id: str | None, lookup_server_reachability: LookupServerReachability
 ) -> tuple[Literal["unscoped", "interactive", "m2m", "stale"], MCPServer | None]:
-    if flow.resource_server_id is None:
+    if server_id is None:
         return "unscoped", None
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415  # import cycle
         MCPServerManager,
         global_mcp_server_manager,
     )
 
-    server: Final = global_mcp_server_manager.get_mcp_server_by_id(flow.resource_server_id)
+    server: Final = global_mcp_server_manager.get_mcp_server_by_id(server_id)
     if (
         server is None
         or not (server.is_gateway_managed_oauth2 or server.advertises_gateway_authorization_server)
-        or not await lookup_server_reachability(flow.user_id, server.server_id)
+        or not await lookup_server_reachability(user_id, server.server_id)
     ):
         return "stale", None
     state: Final = (
@@ -847,6 +870,7 @@ class ConnectFlowDescription(TypedDict):
     server_id: ReadOnly[str | None]
     server_name: ReadOnly[str | None]
     connected: ReadOnly[bool | None]
+    requested_scopes: NotRequired[ReadOnly[tuple[str, ...]]]
 
 
 async def _describe_opened_flow(
@@ -854,19 +878,11 @@ async def _describe_opened_flow(
     lookup_vendor_credential: LookupVendorCredential,
     lookup_server_reachability: LookupServerReachability,
 ) -> ConnectFlowDescription | Response:
-    state, server = await _flow_target(flow, lookup_server_reachability)
+    if flow.upstream_scopes:
+        return await _describe_upstream_scope_flow(flow, lookup_vendor_credential, lookup_server_reachability)
+    state, server = await _flow_target(flow.user_id, flow.resource_server_id, lookup_server_reachability)
     if state == "interactive" and server is not None:
-        credential: Final = await lookup_vendor_credential(flow.user_id, server.server_id)
-        if credential == "unavailable":
-            return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION)
-        interactive_description: Final[ConnectFlowDescription] = {
-            "state": state,
-            "client_origin": _origin_only(flow.redirect_uri),
-            "server_id": server.server_id,
-            "server_name": server.server_name or server.alias or server.name,
-            "connected": credential == "present",
-        }
-        return interactive_description
+        return await _describe_interactive_target(flow, server, flow.requested_scopes, lookup_vendor_credential)
     described: Final[ConnectFlowDescription] = {
         "state": state,
         "client_origin": _origin_only(flow.redirect_uri),
@@ -877,6 +893,70 @@ async def _describe_opened_flow(
     return described
 
 
+async def _describe_interactive_target(
+    flow: _ConnectFlow,
+    server: MCPServer,
+    scopes: tuple[str, ...],
+    lookup_vendor_credential: LookupVendorCredential,
+) -> ConnectFlowDescription | Response:
+    credential: Final = (
+        await lookup_vendor_credential(flow.user_id, server.server_id, scopes)
+        if scopes
+        else await lookup_vendor_credential(flow.user_id, server.server_id)
+    )
+    if credential == "unavailable":
+        return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION)
+    described: Final[ConnectFlowDescription] = {
+        "state": "interactive",
+        "client_origin": _origin_only(flow.redirect_uri),
+        "server_id": server.server_id,
+        "server_name": server.server_name or server.alias or server.name,
+        "connected": credential == "present",
+    }
+    return {**described, "requested_scopes": scopes} if scopes else described
+
+
+async def _describe_upstream_scope_flow(
+    flow: _ConnectFlow,
+    lookup_vendor_credential: LookupVendorCredential,
+    lookup_server_reachability: LookupServerReachability,
+) -> ConnectFlowDescription | Response:
+    server_ids: Final = tuple(dict.fromkeys(item.server_id for item in flow.upstream_scopes))
+    targets: Final = tuple(
+        [await _flow_target(flow.user_id, server_id, lookup_server_reachability) for server_id in server_ids]
+    )
+    if any(state != "interactive" or server is None for state, server in targets):
+        return _oauth_error(400, "invalid_scope", "an upstream permission target is unavailable")
+    descriptions: Final = tuple(
+        [
+            await _describe_interactive_target(
+                flow,
+                server,
+                tuple(
+                    dict.fromkeys(
+                        item.permission for item in flow.upstream_scopes if item.server_id == server.server_id
+                    )
+                ),
+                lookup_vendor_credential,
+            )
+            for _, server in targets
+            if server is not None
+        ]
+    )
+    for description in descriptions:
+        if isinstance(description, Response) or not description["connected"]:
+            return description
+    if flow.resource_server_id is not None:
+        return descriptions[0]
+    return {
+        "state": "unscoped",
+        "client_origin": _origin_only(flow.redirect_uri),
+        "server_id": None,
+        "server_name": None,
+        "connected": None,
+    }
+
+
 async def describe_connect_flow(
     request: Request,
     flow_handle: str,

litellm/proxy/_experimental/mcp_server/legacy_callbacks.py

@@ -1,6 +1,7 @@
 from collections.abc import Mapping
 from typing import Final, Protocol
 
+import httpx2
 from mcp.client.session import ClientRequestContext
 from mcp.types import (
     CreateMessageRequestParams,
@@ -14,6 +15,11 @@ from mcp.types import (
 from litellm.constants import MCP_CLIENT_TIMEOUT
 from litellm.proxy._experimental.mcp_server.contracts import OperationContext
 from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx
+from litellm.proxy._experimental.mcp_server.scope_challenge import (
+    OAuthScopeResponse,
+    is_tool_call_request,
+    scope_response_from_request,
+)
 from litellm.proxy._types import UserAPIKeyAuth
 
 
@@ -100,3 +106,19 @@ def create_elicitation_callback(timeout: float | None = None) -> ElicitationCall
         )
 
     return callback
+
+
+def get_scope_response() -> OAuthScopeResponse | None:
+    context: Final = get_active_mcp_request_ctx()
+    return scope_response_from_request(context.request) if context is not None else None
+
+
+async def record_upstream_tool_authorization(response: httpx2.Response, *, pending: OAuthScopeResponse | None) -> None:
+    if pending is None or not pending.pending or not response.is_success or response.request.method != "POST":
+        return
+    try:
+        is_tool_call: Final = is_tool_call_request(response.request.content)
+    except httpx2.RequestNotRead:
+        return
+    if is_tool_call:
+        pending.allow()

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -38,6 +38,7 @@ import httpx2
 from fastapi import HTTPException
 from httpx import HTTPStatusError
 from mcp import ReadResourceResult, Resource
+from mcp.shared.auth import OAuthMetadata
 from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
 from mcp.types import (
     CallToolResult,
@@ -57,7 +58,7 @@ from mcp.types import (
     ResourceTemplate,
 )
 from mcp.types import Tool as MCPTool
-from pydantic import AnyUrl, Field, SecretStr, TypeAdapter
+from pydantic import AnyUrl, Field, SecretStr, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, assert_never
 
 import litellm
@@ -103,9 +104,10 @@ from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
     ServerListFault,
     raise_classified_list_failure,
     upstream_auth_challenge,
+    upstream_insufficient_scope,
 )
 from litellm.proxy._experimental.mcp_server.interactions import bind_target
-from litellm.proxy._experimental.mcp_server.legacy_callbacks import has_active_mcp_request_context
+from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, has_active_mcp_request_context
 from litellm.proxy._experimental.mcp_server.mcp_debug import describe_upstream_http_failure, record_auth_resolution
 from litellm.proxy._experimental.mcp_server.oauth2_token_cache import (
     MCPPerUserTokenCache,
@@ -895,6 +897,10 @@ def carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_serve
         new_server.authorization_response_iss_parameter_supported = (  # rebind-ok: publish on the existing rebuild object
             previous_server.authorization_response_iss_parameter_supported
         )
+    if may_carry and new_server.issuer == previous_server.issuer and new_server.authorization_server_metadata is None:
+        new_server.authorization_server_metadata = (  # rebind-ok: publish capability on the existing rebuild object
+            previous_server.authorization_server_metadata
+        )
     if new_server.authorization_url is None and previous_server.authorization_url:
         new_server.authorization_url = previous_server.authorization_url
     if may_carry and new_server.token_url is None and previous_server.token_url:
@@ -930,7 +936,12 @@ def _restrict_discovery_to_corroborated_authorization_server(
         return metadata
     if _endpoints_corroborate_authorization_url(metadata.authorization_url, manual_authorization_url):
         return metadata
-    if not metadata.token_url and not metadata.registration_url and not metadata.client_id_metadata_document_supported:
+    if (
+        not metadata.token_url
+        and not metadata.registration_url
+        and not metadata.client_id_metadata_document_supported
+        and metadata.authorization_server_metadata is None
+    ):
         return metadata
     bridge_note: Final = (
         " The discovered registration_url is rejected with it, so this dcr_bridge server stays on the"
@@ -949,7 +960,12 @@ def _restrict_discovery_to_corroborated_authorization_server(
         bridge_note,
     )
     return metadata.model_copy(
-        update={"token_url": None, "registration_url": None, "client_id_metadata_document_supported": False}
+        update={
+            "token_url": None,
+            "registration_url": None,
+            "client_id_metadata_document_supported": False,
+            "authorization_server_metadata": None,
+        }
     )
 
 
@@ -2086,6 +2102,7 @@ class MCPServerManager:
         return server.model_copy(
             update={
                 "scopes": server.scopes or metadata.scopes,
+                "authorization_server_metadata": metadata.authorization_server_metadata,
                 "issuer": server.issuer or discovered_issuer,
                 "client_id_metadata_document_supported": metadata.client_id_metadata_document_supported,
                 "authorization_response_iss_parameter_supported": (
@@ -2678,6 +2695,9 @@ class MCPServerManager:
                 scopes=resolved_scopes,
                 configured_scopes=tuple(configured_scopes) if configured_scopes else None,
                 issuer=effective_issuer,
+                authorization_server_metadata=(
+                    gated_oauth_metadata.authorization_server_metadata if gated_oauth_metadata else None
+                ),
                 client_id_metadata_document_supported=(
                     gated_oauth_metadata.client_id_metadata_document_supported if gated_oauth_metadata else None
                 ),
@@ -3259,6 +3279,9 @@ class MCPServerManager:
             scopes=resolved_scopes,
             configured_scopes=configured_scopes,
             issuer=effective_issuer,
+            authorization_server_metadata=(
+                gated_oauth_metadata.authorization_server_metadata if gated_oauth_metadata else None
+            ),
             client_id_metadata_document_supported=(
                 gated_oauth_metadata.client_id_metadata_document_supported if gated_oauth_metadata else None
             ),
@@ -5383,7 +5406,16 @@ class MCPServerManager:
                 continue
 
             scopes = self._extract_scopes(data.get("scopes_supported"))
+            try:
+                authorization_metadata = OAuthMetadata.model_validate(data)
+            except ValidationError:
+                authorization_metadata = None
             metadata = MCPOAuthMetadata(
+                authorization_server_metadata=(
+                    TypeAdapter(dict[str, object]).validate_python(authorization_metadata.model_dump(mode="json"))
+                    if authorization_metadata is not None
+                    else None
+                ),
                 scopes=scopes,
                 authorization_url=data.get("authorization_endpoint"),
                 token_url=data.get("token_endpoint"),
@@ -6491,11 +6523,12 @@ class MCPServerManager:
             # oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not
             # added here even though the list path still relays for it.
             relays_upstream_auth: Final = mcp_server.is_client_forwarded_token
+            managed_per_user: Final = mcp_server.is_gateway_managed_oauth2 and mcp_server.needs_user_oauth_token
             server_label: Final = mcp_server.name or mcp_server.server_name or mcp_server.alias or ""
 
             async def _call_tool_via_client(client, params):
                 async with self._outbound_limit(mcp_server, persistent_session):
-                    if not relays_upstream_auth:
+                    if not relays_upstream_auth and not managed_per_user:
                         return await client.call_tool(
                             params,
                             host_progress_callback=host_progress_callback,
@@ -6520,8 +6553,17 @@ class MCPServerManager:
                             persistent_session=persistent_session,
                         )
                     except Exception as e:
+                        required_scope: Final = upstream_insufficient_scope(e) if managed_per_user else None
+                        if required_scope is not None:
+                            raise MCPUpstreamAuthError(
+                                status_code=403,
+                                www_authenticate=None,
+                                server_name=mcp_server.alias or server_label,
+                                required_scope=required_scope,
+                                server_id=mcp_server.server_id,
+                            ) from e
                         auth_info: Final = _extract_upstream_auth_failure(e)
-                        if auth_info is None or auth_info[0] != 401:
+                        if not relays_upstream_auth or auth_info is None or auth_info[0] != 401:
                             # A genuine (non-auth or 403-forbidden) upstream/transport failure.
                             # raise_on_error demoted the client-layer log to debug, so surface it here at
                             # warning level to keep the outage visible; the caller still gets the graceful
@@ -6866,6 +6908,15 @@ class MCPServerManager:
             )
             tasks.append(during_hook_task)
 
+        scope_response: Final = get_scope_response()
+        if scope_response is not None and not (
+            mcp_server.is_gateway_managed_oauth2
+            and mcp_server.needs_user_oauth_token
+            and not mcp_server.spec_path
+            and mcp_server.transport in (MCPTransport.http, MCPTransport.sse)
+        ):
+            scope_response.allow()
+
         caller_oauth2_headers: Final = oauth2_headers
         oauth2_headers = await self._resolve_oauth2_headers_for_tool_call(mcp_server, oauth2_headers, user_api_key_auth)
 

litellm/proxy/_experimental/mcp_server/oauth_utils.py

@@ -2,6 +2,8 @@
 (BYOK + discoverable / pass-through OAuth proxy)."""
 
 import os
+import re
+from dataclasses import dataclass
 from ipaddress import ip_address
 from typing import TYPE_CHECKING, Final, NoReturn
 from urllib.parse import ParseResult, urlparse, urlsplit, urlunparse, urlunsplit
@@ -819,3 +821,32 @@ def build_upstream_oauth2_token_request(
     if not resource:
         return client_auth
     return TokenEndpointClientAuth(headers=client_auth.headers, body={**client_auth.body, "resource": resource})
+
+
+UPSTREAM_SCOPE_PREFIX: Final = "litellm:mcp_scope:"
+
+
+@dataclass(frozen=True, slots=True)
+class UpstreamScope:
+    server_id: str
+    permission: str
+
+
+def encode_upstream_scope(server_id: str, permission: str) -> str:
+    return f"{UPSTREAM_SCOPE_PREFIX}{server_id.encode().hex()}:{permission.encode().hex()}"
+
+
+def decode_upstream_scope(token: str) -> UpstreamScope | None:
+    encoded: Final = token.removeprefix(UPSTREAM_SCOPE_PREFIX).split(":")
+    if not token.startswith(UPSTREAM_SCOPE_PREFIX) or len(encoded) != 2:
+        return None
+    try:
+        server_id: Final = bytes.fromhex(encoded[0]).decode()
+        permission: Final = bytes.fromhex(encoded[1]).decode()
+    except (ValueError, UnicodeError):
+        return None
+    if not server_id or re.fullmatch(r"[!#-\[\]-~]+", permission) is None:
+        return None
+    if encode_upstream_scope(server_id, permission) != token:
+        return None
+    return UpstreamScope(server_id, permission)

litellm/proxy/_experimental/mcp_server/operations.py

@@ -94,6 +94,7 @@ from litellm.proxy._experimental.mcp_server.interactions import (
     seal_continuation,
     target_digest,
 )
+from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response
 from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: F401  # legacy module exports
     MCPServerManager,
     _caller_authorization_fans_out,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
@@ -3009,11 +3010,12 @@ async def _execute_mcp_server_tool_call(
             is_error=True,
         )
     except MCPUpstreamAuthError as e:
-        # The MCP session manager serializes handler exceptions as JSON-RPC errors, so a
-        # mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST
-        # call path and the connect-time preemptive check do. Return an explicit isError
-        # naming the upstream status (at info level, not a traceback) so the client still
-        # learns it must re-authenticate upstream and expected pass-through 401s don't spam.
+        scope_response: Final = get_scope_response()
+        if scope_response is not None:
+            scope_response.deny(e)
+        # The request-owned response adapter relays managed scope challenges before
+        # Streamable HTTP headers commit. Other auth failures retain the existing
+        # explicit tool error, including failures after streaming has begun.
         verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code)
         return CallToolResult(
             content=[

litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py

@@ -65,7 +65,7 @@ def _parse_expires_in(raw: object) -> int | None:
 
 
 def _parse_scopes(raw: object) -> tuple[str, ...] | None:
-    return tuple(raw.split()) if isinstance(raw, str) and raw else None
+    return tuple(raw.split()) if isinstance(raw, str) else None
 
 
 class AuthorizationCodeRefresher:
@@ -156,7 +156,8 @@ class AuthorizationCodeRefresher:
         rotated: Final = body.get("refresh_token")
         new_refresh: Final = rotated if isinstance(rotated, str) and rotated else token.refresh_token
         expires_in: Final = _parse_expires_in(body.get("expires_in"))
-        scopes: Final = _parse_scopes(body.get("scope")) or token.scopes
+        granted_scopes: Final = _parse_scopes(body.get("scope"))
+        scopes: Final = granted_scopes if granted_scopes is not None else token.scopes
 
         await self._persist(
             user_id,
@@ -164,7 +165,7 @@ class AuthorizationCodeRefresher:
             access_token,
             new_refresh,
             expires_in,
-            scopes or None,
+            scopes,
             **({"identity_binding_proof": binding_proof} if binding_proof is not None else {}),
             **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}),
         )

litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py

@@ -88,7 +88,7 @@ async def _persist_credential(
         access_token=access_token,
         refresh_token=refresh_token,
         expires_in=expires_in,
-        scopes=list(scopes) if scopes else None,
+        scopes=list(scopes) if scopes is not None else None,
         skip_byok_guard=True,
         identity_binding_proof=identity_binding_proof,
         **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}),