Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_add-claude-sonnet-5-pricing

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,38 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
+        return 0
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/down.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0

.github/e2e-stack/secrets_to_env.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
+
+
+def main() -> int:
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
+        return 1
+    for value in secrets.values():
+        if len(value) >= MIN_MASKED_LENGTH:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -0,0 +1,44 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/workflows/_test-unit-base.yml

@@ -55,17 +55,14 @@ on:
 permissions:
   contents: read
 
+env:
+  UV_PYTHON: "3.12"
+
 jobs:
   run:
-    name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }}
+    name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.job-timeout-minutes }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-    env:
-      UV_PYTHON: ${{ matrix.python-version }}
     permissions:
       contents: read
       pull-requests: read
@@ -88,7 +85,7 @@ jobs:
         timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.UV_PYTHON }}
 
       - name: Set up uv
         if: steps.changes.outputs.decision != 'skip'
@@ -103,9 +100,9 @@ jobs:
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
+          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-
+            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
 
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
@@ -139,7 +136,7 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
           DIST: ${{ inputs.dist }}
-          COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }}
+          COVERAGE_CORE: sysmon
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \
@@ -166,7 +163,7 @@ jobs:
           fi
 
       - name: Save coverage report
-        if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip'
+        if: always() && steps.changes.outputs.decision != 'skip'
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/report-rust-release-wheel.yml

@@ -24,8 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
-      issues: write # PR comments use the issues API
-      pull-requests: read # Current-head validation rejects stale workflow runs
+      pull-requests: write
 
     steps:
       - name: Link release wheel report on PR

.github/workflows/test-code-quality.yml

@@ -74,6 +74,9 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-e2e-changed.yml

@@ -0,0 +1,240 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
+          tests="$(printf '%s\n' "${files}" \
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no changed e2e test files supported by this stack; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          umask 077
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        id: boot
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
+        run: |
+          umask 077
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
+            status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
+            set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
+            echo "pass ${pass} of 3 passed"
+          done
+
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi

.github/workflows/test-terraform-modules.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
   pull_request:
     branches:
@@ -13,6 +14,7 @@ on:
       - "litellm_**"
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
 
 permissions:
@@ -52,3 +54,32 @@ jobs:
       # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
       - name: test
         run: terraform test
+
+  gcp-module:
+    name: fmt, validate, test (gcp)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/gcp
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      - name: test
+        run: terraform test

CLAUDE.md

@@ -52,7 +52,7 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
 
 Python max line length is 120, not 88
 
-When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
+Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on `litellm_internal_staging` in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. If your branch already carries a budget edit, drop it before opening the PR
 
 `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
 

Dockerfile

@@ -67,6 +67,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -89,6 +90,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

basedpyright-code-budget.json

@@ -105,13 +105,13 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38283
+    "limit": 38271
   },
   "reportUnknownParameterType": {
     "limit": 19584
   },
   "reportUnknownVariableType": {
-    "limit": 29829
+    "limit": 29814
   },
   "reportUnnecessaryCast": {
     "limit": 110

ci_cd/generate_model_prices_schema.py

@@ -58,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_audio_formats": {
+        "type": "array",
+        "description": "Audio container formats the model can return.",
+        "items": {"type": "string", "enum": ["mp3", "wav"]},
+    },
     "supported_endpoints": {
         "type": "array",
         "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
@@ -231,6 +236,10 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
         },
         "comment": STRING,
         "audio_transcription_config": STRING,
+        "vertex_ai_audio_api": {
+            "type": "string",
+            "enum": ["lyria_predict", "lyria_interactions"],
+        },
     }
 
 

docker/Dockerfile.database

@@ -65,6 +65,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -87,6 +88,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

docker/Dockerfile.non_root

@@ -71,6 +71,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -99,6 +100,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -109,6 +111,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13; \
     fi
 

gateway/Dockerfile

@@ -47,6 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -59,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

litellm-proxy-extras/litellm_proxy_extras/migrations/20260905120000_add_per_server_oauth_discovery_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "per_server_oauth_discovery" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -343,6 +343,7 @@ model LiteLLM_MCPServerTable {
   delegate_auth_to_upstream Boolean @default(false)
   oauth_passthrough Boolean @default(false)
   dcr_bridge Boolean?
+  per_server_oauth_discovery Boolean @default(false)
   is_byok               Boolean  @default(false)
   byok_description      String[] @default([])
   byok_api_key_help_url String?

litellm/_lazy_imports.py

@@ -229,7 +229,7 @@ def _module_attribute(module: ModuleType, attr_name: str) -> object:
     return attribute["value"]
 
 
-def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], category: str) -> object:
+def _generic_lazy_import(name: str, import_map: Mapping[str, tuple[str, str]], category: str) -> object:
     """
     Generic function that handles lazy importing for most attributes.
 

litellm/batches/batch_utils.py

@@ -25,6 +25,27 @@ class BatchCostUsageResult:
     failed_requests: int
 
 
+_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
+_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
+
+
+def batch_cost_is_final(batch: Batch) -> bool:
+    """Whether this retrieve of the batch is the one to account its cost from.
+
+    A batch still in flight has nothing to price, and a "completed" batch can report
+    no output_file_id for a moment before the output populates; pricing either records
+    $0 under the batch's single spend row and pins it there. Final means a completed
+    batch whose output file has arrived or whose counts prove no line succeeded, or
+    any other terminal status (failed, cancelled, expired).
+    """
+    if batch.status not in _TERMINAL_BATCH_STATUSES:
+        return False
+    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
+        return True
+    request_counts: Final = batch.request_counts
+    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
+
+
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: list[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],

litellm/cost_calculator.py

@@ -81,6 +81,7 @@ from litellm.llms.together_ai.cost_calculator import (
     get_model_params_and_category,
     has_together_registry_pricing,
 )
+from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -496,6 +497,13 @@ def cost_per_token(
 
     # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
     if call_type == "speech" or call_type == "aspeech":
+        lyria_generation_cost: Final = (
+            get_vertex_ai_lyria_generation_cost(model=model_without_prefix)
+            if custom_llm_provider in ("vertex_ai", "vertex_ai_beta")
+            else None
+        )
+        if lyria_generation_cost is not None:
+            return 0.0, lyria_generation_cost
         speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider)
         cost_metric: Final = select_cost_metric_for_model(speech_model_info)
         prompt_cost: float = 0.0

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -16,23 +16,32 @@ import asyncio
 import os
 import time
 import traceback
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeVar
 from urllib.parse import urlparse
 
 from litellm._logging import verbose_logger
+from litellm.integrations.batch_utils import (
+    BatchSendCancelled,
+    send_batch_with_413_split,
+    undelivered_after_http_error,
+)
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
+    MaskedHTTPStatusError,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.types.integrations.azure_sentinel import AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
 from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 DEFAULT_AZURE_AUTHORITY_HOST: Final = "https://login.microsoftonline.com"
 DEFAULT_AZURE_MONITOR_SCOPE: Final = "https://monitor.azure.com/.default"
 
+_QueuedPayload = TypeVar("_QueuedPayload", StandardLoggingPayload, StandardAuditLogPayload)
+
 MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
     {
         "login.microsoftonline.com": DEFAULT_AZURE_MONITOR_SCOPE,
@@ -153,6 +162,8 @@ class AzureSentinelLogger(CustomBatchLogger):
         asyncio.create_task(self.periodic_flush())
         self.log_queue: list[StandardLoggingPayload] = []
         self.audit_log_queue: list[StandardAuditLogPayload] = []
+        self.logs_awaiting_retry = False
+        self.audit_logs_awaiting_retry = False
 
     @staticmethod
     def _normalize_authority_host(authority_host: str) -> str:
@@ -245,8 +256,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -275,8 +286,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -298,12 +309,24 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.audit_log_queue.append(audit_log)
 
-            if len(self.audit_log_queue) >= self.batch_size:
-                await self.async_send_audit_batch()
+            if len(self.audit_log_queue) >= self.batch_size and not self.audit_logs_awaiting_retry:
+                await self._threshold_send_audit_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Audit Log Layer Error - %s\n%s", e, traceback.format_exc())
 
+    async def _threshold_send_logs(self) -> None:
+        async with self.flush_lock:
+            if self.logs_awaiting_retry or len(self.log_queue) < self.batch_size:
+                return
+            await self.async_send_batch()
+
+    async def _threshold_send_audit_logs(self) -> None:
+        async with self.flush_lock:
+            if self.audit_logs_awaiting_retry or len(self.audit_log_queue) < self.batch_size:
+                return
+            await self.async_send_audit_batch()
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -311,67 +334,110 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.log_queue,
-            api_endpoint=self.api_endpoint,
-            log_type="logs",
-        )
+        batch_to_send: Final = tuple(self.log_queue)
+        self.log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.api_endpoint,
+                log_type="logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.log_queue = self._requeue(cancelled.undelivered, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.log_queue = self._requeue(batch_to_send, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise
+        self.log_queue = self._requeue(undelivered, self.log_queue, "logs")
+        self.logs_awaiting_retry = bool(undelivered) and bool(self.log_queue)
 
     async def async_send_audit_batch(self):
         """
         Sends the batch of audit logs to Azure Monitor Logs Ingestion API
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.audit_log_queue,
-            api_endpoint=self.audit_api_endpoint,
-            log_type="audit logs",
+        batch_to_send: Final = tuple(self.audit_log_queue)
+        self.audit_log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.audit_api_endpoint,
+                log_type="audit logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.audit_log_queue = self._requeue(cancelled.undelivered, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.audit_log_queue = self._requeue(batch_to_send, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise
+        self.audit_log_queue = self._requeue(undelivered, self.audit_log_queue, "audit logs")
+        self.audit_logs_awaiting_retry = bool(undelivered) and bool(self.audit_log_queue)
+
+    def _requeue(
+        self,
+        undelivered: tuple[_QueuedPayload, ...],
+        queue: list[_QueuedPayload],
+        log_type: str,
+    ) -> list[_QueuedPayload]:
+        merged: Final = [*undelivered, *queue]  # mutable-ok: queue trimming returns a mutable logger queue
+        overflow: Final = len(merged) - self.max_queue_size
+        if overflow <= 0:
+            return merged
+
+        verbose_logger.warning(
+            "Azure Sentinel: %s queue exceeded max_queue_size=%s, dropped %s oldest records",
+            log_type,
+            self.max_queue_size,
+            overflow,
         )
+        return merged[overflow:]
 
     async def _async_send_batch_to_api(
         self,
-        log_queue: list[StandardLoggingPayload | StandardAuditLogPayload],
+        log_queue: tuple[_QueuedPayload, ...],
         api_endpoint: str,
         log_type: str,
-    ) -> None:
+    ) -> tuple[_QueuedPayload, ...]:
+        if not log_queue:
+            return ()
+
+        verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
         try:
-            if not log_queue:
-                return
-
-            verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
-
-            # Get OAuth2 token
             bearer_token: Final = await self._get_oauth_token()
+        except MaskedHTTPStatusError as e:
+            return undelivered_after_http_error(log_queue, e.status_code, "Azure Sentinel OAuth token", str(e))
+        except Exception as e:
+            verbose_logger.exception("Azure Sentinel Error getting OAuth token - %s", e)
+            return tuple(log_queue)
 
-            # Convert log queue to JSON array format expected by Logs Ingestion API
-            # Each log entry should be a JSON object in the array
-            body: Final = safe_dumps(log_queue)
+        headers: Final = {
+            "Authorization": f"Bearer {bearer_token}",
+            "Content-Type": "application/json",
+        }
 
-            # Set headers for Logs Ingestion API
-            headers: Final = {
-                "Authorization": f"Bearer {bearer_token}",
-                "Content-Type": "application/json",
-            }
-
-            # Send the request
-            response = await self.async_httpx_client.post(url=api_endpoint, data=body.encode("utf-8"), headers=headers)
-
-            if response.status_code not in [200, 204]:
-                verbose_logger.error(
-                    "Azure Sentinel API error: status_code=%s, response=%s",
-                    response.status_code,
-                    response.text,
-                )
-                raise Exception(f"Failed to send logs to Azure Sentinel: {response.status_code} - {response.text}")
-
-            verbose_logger.debug(
-                "Azure Sentinel: Response from API status_code: %s",
-                response.status_code,
+        async def _send_batch(batch: Sequence[_QueuedPayload]):
+            body: Final = safe_dumps(batch)
+            return await self.async_httpx_client.post(
+                url=api_endpoint,
+                data=body.encode("utf-8"),
+                headers=headers,
             )
 
-        except Exception as e:
-            verbose_logger.exception("Azure Sentinel Error sending batch API - %s\n%s", e, traceback.format_exc())
-        finally:
-            log_queue.clear()
+        return await send_batch_with_413_split(
+            batch=log_queue,
+            send_batch=_send_batch,
+            exceeds_limits=lambda batch: (
+                len(batch) > self.batch_size
+                or len(safe_dumps(batch).encode("utf-8")) > AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
+            ),
+            success_status_codes=frozenset({200, 204}),
+            integration_name="Azure Sentinel",
+            drop_error_message="Azure Sentinel API Error - Payload too large for a single record",
+            non_success_handler=undelivered_after_http_error,
+        )
 
     async def flush_queue(self):
         if self.flush_lock is None:

litellm/integrations/batch_utils.py

@@ -0,0 +1,160 @@
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from typing import Final, Generic, TypeVar
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
+
+_BatchItem = TypeVar("_BatchItem")
+
+_RETRYABLE_CLIENT_STATUS_CODES: Final = frozenset({408, 429})
+
+
+def is_retryable_status(status_code: int) -> bool:
+    return not 400 <= status_code < 500 or status_code in _RETRYABLE_CLIENT_STATUS_CODES
+
+
+def undelivered_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    """The records to requeue after a non-2xx: all of them on a status a retry can clear, none on
+    a 4xx that would only repeat, since retaining those retries a misconfiguration forever."""
+    if is_retryable_status(status_code):
+        verbose_logger.error(
+            "%s API error: status_code=%s, will retry %s records - %s",
+            integration_name,
+            status_code,
+            len(batch),
+            detail,
+        )
+        return tuple(batch)
+    verbose_logger.error(
+        "%s API error: status_code=%s is not retryable, dropped %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return ()
+
+
+def requeue_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    verbose_logger.error(
+        "%s API error: status_code=%s, will retry %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return tuple(batch)
+
+
+class BatchSendCancelled(asyncio.CancelledError, Generic[_BatchItem]):
+    """Cancellation of a batch send, carrying only the records the destination never accepted.
+
+    A batch split under the size cap is delivered in pieces, so requeueing all of it after a
+    cancellation partway through would send the accepted pieces a second time.
+    """
+
+    def __init__(self, undelivered: tuple[_BatchItem, ...]) -> None:
+        super().__init__()
+        self.undelivered: Final = undelivered
+
+
+async def _keep_the_remainder_on_cancel(
+    send: Awaitable[tuple[_BatchItem, ...]],
+    remainder: Sequence[_BatchItem],
+) -> tuple[_BatchItem, ...]:
+    try:
+        return await send
+    except BatchSendCancelled as cancelled:
+        raise BatchSendCancelled((*cancelled.undelivered, *remainder)) from cancelled
+
+
+async def send_batch_with_413_split(
+    batch: Sequence[_BatchItem],
+    send_batch: Callable[[Sequence[_BatchItem]], Awaitable[httpx.Response]],
+    exceeds_limits: Callable[[Sequence[_BatchItem]], bool],
+    success_status_codes: frozenset[int],
+    integration_name: str,
+    drop_error_message: str,
+    non_success_handler: Callable[
+        [Sequence[_BatchItem], int, str, str], tuple[_BatchItem, ...]
+    ] = requeue_after_http_error,
+) -> tuple[_BatchItem, ...]:
+    async def _halve() -> tuple[_BatchItem, ...]:
+        midpoint: Final = len(batch) // 2
+        left_batch: Final = batch[:midpoint]
+        right_batch: Final = batch[midpoint:]
+        left_undelivered: Final = await _keep_the_remainder_on_cancel(
+            send_batch_with_413_split(
+                batch=left_batch,
+                send_batch=send_batch,
+                exceeds_limits=exceeds_limits,
+                success_status_codes=success_status_codes,
+                integration_name=integration_name,
+                drop_error_message=drop_error_message,
+                non_success_handler=non_success_handler,
+            ),
+            right_batch,
+        )
+        if left_undelivered:
+            return (*left_undelivered, *right_batch)
+        return await send_batch_with_413_split(
+            batch=right_batch,
+            send_batch=send_batch,
+            exceeds_limits=exceeds_limits,
+            success_status_codes=success_status_codes,
+            integration_name=integration_name,
+            drop_error_message=drop_error_message,
+            non_success_handler=non_success_handler,
+        )
+
+    async def _handle_413() -> tuple[_BatchItem, ...]:
+        if len(batch) == 1:
+            verbose_logger.error(drop_error_message)
+            return ()
+        return await _halve()
+
+    if not batch:
+        return ()
+
+    try:
+        oversized: Final = exceeds_limits(batch)
+    except Exception as e:  # noqa: BLE001  # any record that cannot be serialized is isolated and dropped alone
+        if len(batch) > 1:
+            return await _halve()
+        verbose_logger.exception("%s dropped a record that cannot be serialized - %s", integration_name, e)
+        return ()
+    if oversized and len(batch) > 1:
+        return await _halve()
+
+    try:
+        response: Final = await send_batch(batch)
+    except MaskedHTTPStatusError as e:
+        if e.status_code == 413:
+            return await _handle_413()
+        return non_success_handler(batch, e.status_code, integration_name, str(e))
+    except asyncio.CancelledError as cancelled:
+        raise BatchSendCancelled(tuple(batch)) from cancelled
+    except Exception as e:
+        verbose_logger.exception("%s Error sending batch API - %s", integration_name, e)
+        return tuple(batch)
+
+    if response.status_code == 413:
+        return await _handle_413()
+    if response.status_code not in success_status_codes:
+        return non_success_handler(batch, response.status_code, integration_name, response.text)
+
+    verbose_logger.debug("%s delivered %s records, status_code=%s", integration_name, len(batch), response.status_code)
+    return ()

litellm/integrations/custom_guardrail.py

@@ -948,6 +948,23 @@ class CustomGuardrail(CustomLogger):
         """
         return False
 
+    def _suppressed_by_auto_router_compression(self) -> bool:
+        """True when an auto router's own compression policy suppresses this guardrail.
+
+        Reads request-scoped state set by `arm_pre_call`, never request metadata. The
+        caller controls metadata, and metadata reaches spend logs the caller can read,
+        so a suppression list carried there would be one a request could replay to
+        switch off a PII or content-filter guardrail for itself.
+        """
+        name: Final = self.guardrail_name
+        if not name:
+            return False
+        from litellm.proxy.guardrails.auto_router_compression import (
+            suppressed_compression_guardrails,
+        )
+
+        return name in suppressed_compression_guardrails()
+
     def should_run_guardrail(
         self,
         data,
@@ -956,6 +973,9 @@ class CustomGuardrail(CustomLogger):
         """
         Returns True if the guardrail should be run on the event_type
         """
+        if self._suppressed_by_auto_router_compression():
+            return False
+
         requested_guardrails: Final = self.get_guardrail_from_metadata(data)
         disable_global_guardrail: Final = self.get_disable_global_guardrail(data)
         opted_out_global_guardrails: Final = self.get_opted_out_global_guardrails_from_metadata(data)

litellm/integrations/datadog/datadog.py

@@ -29,6 +29,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.integrations.batch_utils import BatchSendCancelled, requeue_after_http_error, send_batch_with_413_split
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.datadog.datadog_handler import (
     get_datadog_base_url_from_env,
@@ -43,7 +44,6 @@ from litellm.integrations.datadog.datadog_mock_client import (
 )
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.llms.custom_httpx.http_handler import (
-    MaskedHTTPStatusError,
     _get_httpx_client,
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -396,6 +396,9 @@ class DataDogLogger(
             if self.is_mock_mode:
                 verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(batch_to_send))
 
+        except BatchSendCancelled as cancelled:
+            self.log_queue = list(cancelled.undelivered) + self.log_queue  # mutable-ok: logger queue remains appendable
+            raise asyncio.CancelledError() from cancelled
         except Exception as e:
             self.log_queue = batch_to_send + self.log_queue
             verbose_logger.exception("Datadog Error sending batch API - %s\n%s", e, traceback.format_exc())
@@ -413,53 +416,16 @@ class DataDogLogger(
         that could not be delivered because of a non-413 (transient) error, so the caller
         re-queues only those and never the events already accepted by Datadog.
         """
-        pending: Final[list[list]] = [batch]
-        while pending:
-            chunk = pending.pop()
-            if not chunk:
-                continue
-            if len(chunk) > 1 and self._exceeds_intake_limits(chunk):
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-            try:
-                response = await self.async_send_compressed_data(chunk)
-            except Exception as e:
-                if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413:
-                    response = e.response
-                else:
-                    verbose_logger.exception("Datadog Error sending batch API - %s", e)
-                    return self._undelivered(chunk, pending)
-
-            if response.status_code == 413:
-                if len(chunk) == 1:
-                    verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value)
-                    continue
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-
-            if response.status_code != 202:
-                verbose_logger.error(
-                    "Datadog: unexpected response status_code=%s, text=%s",
-                    response.status_code,
-                    response.text,
-                )
-                return self._undelivered(chunk, pending)
-
-            verbose_logger.debug(
-                "Datadog: delivered %s events, status_code=%s, text=%s",
-                len(chunk),
-                response.status_code,
-                response.text,
-            )
-        return []
-
-    @staticmethod
-    def _undelivered(chunk: list, pending: list[list]) -> list:
-        return chunk + [event for remaining in reversed(pending) for event in remaining]
+        undelivered: Final = await send_batch_with_413_split(
+            batch=batch,
+            send_batch=self.async_send_compressed_data,
+            exceeds_limits=self._exceeds_intake_limits,
+            success_status_codes=frozenset({202}),
+            integration_name="Datadog",
+            drop_error_message=DD_ERRORS.DATADOG_413_ERROR.value,
+            non_success_handler=requeue_after_http_error,
+        )
+        return list(undelivered)  # mutable-ok: caller prepends records to the logger queue
 
     @staticmethod
     def _exceeds_intake_limits(chunk: Sequence[DatadogPayload]) -> bool:
@@ -606,7 +572,7 @@ class DataDogLogger(
         )
         return dd_payload
 
-    async def async_send_compressed_data(self, data: list) -> Response:
+    async def async_send_compressed_data(self, data: Sequence[DatadogPayload]) -> Response:
         """
         Async helper to send compressed data to datadog self.intake_url
 

litellm/litellm_core_utils/health_check_helpers.py

@@ -6,14 +6,13 @@ import base64
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Final, Literal
 
-from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
+from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, DocumentType
+from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging
     from litellm.types.utils import ImageResponse
 
-# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
-TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
 
 # Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG
 TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC"
@@ -29,6 +28,14 @@ def get_image_file_for_health_check() -> bytes:
     return base64.b64decode(TEST_IMAGE_BASE64)
 
 
+def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
+    from litellm.utils import ProviderConfigManager
+
+    provider: Final = next((known for known in LlmProviders if known.value == custom_llm_provider), None)
+    config: Final = ProviderConfigManager.get_provider_ocr_config(model=model, provider=provider) if provider else None
+    return (config or BaseOCRConfig()).get_health_check_document()
+
+
 class HealthCheckHelpers:
     @staticmethod
     async def ahealth_check_wildcard_models(
@@ -247,9 +254,6 @@ class HealthCheckHelpers:
             ),
             "ocr": lambda: litellm.aocr(
                 **_filter_model_params(model_params=model_params),
-                document={
-                    "type": "document_url",
-                    "document_url": TEST_PDF_URL,
-                },
+                document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
             ),
         }

litellm/litellm_core_utils/litellm_logging.py

@@ -36,7 +36,7 @@ from litellm._logging import (
     verbose_logger,
 )
 from litellm._uuid import uuid
-from litellm.batches.batch_utils import _handle_completed_batch
+from litellm.batches.batch_utils import _handle_completed_batch, batch_cost_is_final
 from litellm.caching.caching import DualCache, InMemoryCache
 from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.constants import (
@@ -2899,13 +2899,6 @@ class Logging(LiteLLMLoggingBaseClass):
             ):  # polling job will query these frequently, don't spam db logs
                 return
 
-            from litellm.proxy.openai_files_endpoints.common_utils import (
-                _is_base64_encoded_unified_file_id,
-            )
-
-            # check if file id is a unified file id
-            is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(result.id)
-
             batch_cost: Final = kwargs.get("batch_cost", None)
             batch_usage = kwargs.get("batch_usage", None)
             batch_models = kwargs.get("batch_models", None)
@@ -2913,9 +2906,7 @@ class Logging(LiteLLMLoggingBaseClass):
             batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
             has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
 
-            should_compute_batch_data: Final = (
-                not is_base64_unified_file_id or not has_explicit_batch_data and result.status == "completed"
-            )
+            should_compute_batch_data: Final = not has_explicit_batch_data and batch_cost_is_final(result)
             if has_explicit_batch_data:
                 result._hidden_params["response_cost"] = batch_cost
                 result._hidden_params["batch_models"] = batch_models
@@ -5675,8 +5666,8 @@ class StandardLoggingPayloadSetup:
             error_code=error_status,
             error_class=error_class,
             llm_provider=_llm_provider_in_exception,
-            traceback=traceback_info,
-            error_message=error_message,
+            traceback=_redact_string(traceback_info),
+            error_message=_redact_string(error_message),
             error_rate_limit_category=rate_limit_category,
             error_rate_limit_type=rate_limit_type,
             error_budget_entity_type=budget_error.entity_type if budget_error else None,
@@ -5881,6 +5872,7 @@ def _get_status_fields(
     # Mapping for legacy guardrail status values to new GuardrailStatus values
     GUARDRAIL_STATUS_MAP: Final[dict[str, GuardrailStatus]] = {
         "success": "success",
+        "guardrail_flagged": "guardrail_flagged",
         "blocked": "guardrail_intervened",  # legacy
         "guardrail_intervened": "guardrail_intervened",  # direct
         "failure": "guardrail_failed_to_respond",  # legacy
@@ -5902,6 +5894,7 @@ def _get_status_fields(
     GUARDRAIL_STATUS_SEVERITY: Final[tuple[GuardrailStatus, ...]] = (
         "not_run",
         "success",
+        "guardrail_flagged",
         "guardrail_failed_to_respond",
         "guardrail_intervened",
     )

litellm/llms/anthropic/common_utils.py

@@ -1411,6 +1411,25 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages(  # mutable-ok:
     return [_flatten_web_search_results_in_message(m) for m in messages]  # mutable-ok: JSON wire format
 
 
+def _without_provider_specific_fields(block: object) -> object:
+    if not isinstance(block, dict) or "provider_specific_fields" not in block:
+        return block
+    return {k: v for k, v in block.items() if k != "provider_specific_fields"}  # mutable-ok: JSON wire format
+
+
+def _strip_provider_specific_fields_in_message(message: object) -> object:
+    if not isinstance(message, dict) or not isinstance(message.get("content"), list):
+        return message
+    content: Final = [_without_provider_specific_fields(b) for b in message["content"]]  # mutable-ok: JSON wire format
+    return {**message, "content": content}  # mutable-ok: JSON wire format
+
+
+def strip_provider_specific_fields_from_anthropic_messages(
+    messages: Sequence[object],
+) -> Sequence[object]:
+    return [_strip_provider_specific_fields_in_message(m) for m in messages]  # mutable-ok: JSON wire format
+
+
 def _normalized_cache_control(cache_control: object) -> dict[str, str] | None:  # mutable-ok: JSON wire format
     if not isinstance(cache_control, Mapping):
         return None

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -11,6 +11,7 @@ from typing import (
     Final,
     Literal,
     Protocol,
+    cast,  # noqa: TID251  # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union
     get_args,
 )
 
@@ -100,6 +101,10 @@ class _CombinedChunkSplitter:
     @staticmethod
     def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         choices: Final = _optional_attr_sequence(chunk, "choices")
         if not choices:
             return False
@@ -114,6 +119,7 @@ class _CombinedChunkSplitter:
             or _optional_attr(delta, "tool_calls")
             or _optional_attr(delta, "reasoning_content")
             or _optional_attr(delta, "thinking_blocks")
+            or openai_chat_refusal_text(delta)
         )
 
     _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@@ -305,6 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         # Synthesized compaction block from compact_20260112 polyfill (streaming).
         self.compaction_block = compaction_block
         self.iterations_usage = iterations_usage
+        self._refusal_text: str = ""
         self.sent_compaction_block: bool = False
         # Per-phase flags so the compaction block's start/delta/stop events
         # are emitted (and the public state machine is advanced) in
@@ -572,6 +579,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -806,6 +814,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -993,6 +1002,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     def _increment_content_block_index(self):
         self.current_content_block_index += 1
 
+    def _with_refusal_stop_details(
+        self,
+        processed_chunk: ContentBlockDelta | MessageBlockDelta,
+    ) -> ContentBlockDelta | MessageBlockDelta:
+        if processed_chunk.get("type") != "message_delta" or not self._refusal_text:
+            return processed_chunk
+        delta: Final = cast(Mapping[str, object], processed_chunk["delta"])  # cast-ok: keys checked before use
+        if delta.get("stop_reason") == "max_tokens":
+            return processed_chunk
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            refusal_stop_details,
+        )
+
+        return cast(  # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch
+            ContentBlockDelta | MessageBlockDelta,
+            {  # mutable-ok: fresh translation payload; never mutated after construction
+                **processed_chunk,
+                "delta": {  # mutable-ok: fresh message_delta payload; never mutated after construction
+                    **delta,
+                    "stop_reason": "refusal",
+                    "stop_details": refusal_stop_details(self._refusal_text),
+                },
+            },
+        )
+
     @staticmethod
     def _delta_has_content(processed_chunk: Mapping[str, object]) -> bool:
         """Return True if a translated chunk carries a non-empty
@@ -1035,6 +1069,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
         from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1044,6 +1081,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "content", None):
             return False
+        if openai_chat_refusal_text(delta):
+            return False
         if getattr(delta, "reasoning_content", None):
             return False
         # thinking_blocks whose entries are all empty AND unsigned must not
@@ -1067,13 +1106,19 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         - Different content types in the response
         - Specific markers in the content
         """
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
-        # Example logic - customize based on your needs:
-        # If chunk indicates a tool call
         if chunk.choices[0].finish_reason is not None:
             return False
 
+        refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
+        if refusal_text is not None:
+            self._refusal_text = self._refusal_text + refusal_text
+
         (
             block_type,
             content_block_start,

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import (
 from litellm.llms.anthropic.experimental_pass_through.context_management import (
     PolyfillResult,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    openai_chat_refusal_text,
+    refusal_stop_details,
+)
 from litellm.types.llms.anthropic import (
     ANTHROPIC_HOSTED_TOOLS,
     AllAnthropicPassThroughMessageValues,
@@ -1314,6 +1318,8 @@ class LiteLLMAnthropicMessagesAdapter:
                 new_content.append(
                     AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
                 )
+            if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
+                new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
             # Handle tool calls (in parallel to text content)
             if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
                 for tool_call in choice.message.tool_calls:
@@ -1346,7 +1352,7 @@ class LiteLLMAnthropicMessagesAdapter:
                     # Add provider_specific_fields if signature is present
                     if provider_specific_fields:
                         tool_use_block.provider_specific_fields = provider_specific_fields
-                    new_content.append(tool_use_block.model_dump())
+                    new_content.append(tool_use_block.model_dump(exclude_none=True))
 
         return new_content
 
@@ -1472,14 +1478,23 @@ class LiteLLMAnthropicMessagesAdapter:
             choices=response.choices,
             tool_name_mapping=tool_name_mapping,
         )
+        refusal_text: Final = next(
+            (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
+            None,
+        )
 
         if polyfill_result is not None and polyfill_result.compaction_block is not None:
             anthropic_content.insert(0, polyfill_result.compaction_block)
 
         ## extract finish reason
-        anthropic_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
+        translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
             openai_finish_reason=response.choices[0].finish_reason
         )
+        anthropic_finish_reason: Final = (
+            "refusal"
+            if refusal_text is not None and translated_finish_reason != "max_tokens"
+            else translated_finish_reason
+        )
         # extract usage
         usage: Final[Usage] = getattr(response, "usage")
         anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
@@ -1501,6 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter:
             usage=anthropic_usage,
             content=anthropic_content,
             stop_reason=anthropic_finish_reason,
+            stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
         )
 
         applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@@ -1541,7 +1557,9 @@ class LiteLLMAnthropicMessagesAdapter:
                         "signature": thought_sig,
                     }
                 return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
-            elif choice.delta.content is not None and len(choice.delta.content) > 0:
+            elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
+                choice.delta
+            ) is not None:
                 return "text", TextBlock(type="text", text="")
             elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
                 thinking_blocks = choice.delta.thinking_blocks or []
@@ -1613,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter:
         elif reasoning_content:
             return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
         else:
-            return "text_delta", ContentTextBlockDelta(type="text_delta", text=text)
+            refusal_text: Final = "".join(
+                refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
+            )
+            return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)
 
     def translate_streaming_openai_response_to_anthropic(
         self,

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -18,6 +18,7 @@ from litellm.llms.anthropic.common_utils import (
     flatten_unencrypted_web_search_results_in_anthropic_messages,
     sanitize_tool_use_ids_in_anthropic_messages,
     strip_empty_content_blocks_from_anthropic_messages,
+    strip_provider_specific_fields_from_anthropic_messages,
 )
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -650,7 +651,7 @@ def anthropic_messages_handler(
 
     return base_llm_http_handler.anthropic_messages_handler(
         model=model,
-        messages=messages,
+        messages=strip_provider_specific_fields_from_anthropic_messages(messages),
         anthropic_messages_provider_config=anthropic_messages_provider_config,
         anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params),
         _is_async=is_async,

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -1,8 +1,11 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping, Sequence
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints
 
-from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams
+from litellm.types.llms.anthropic import (
+    AnthropicMessagesRequestOptionalParams,
+    AnthropicStopDetails,
+)
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
@@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] |
     return stop_details if isinstance(stop_details, dict) else None
 
 
+def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails:
+    """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``."""
+    return AnthropicStopDetails(type="refusal", category=None, explanation=explanation)
+
+
+def _mapping_field(container: object, key: str) -> object | None:
+    """One key of a raw provider payload, or None when the payload is not a mapping."""
+    if not isinstance(container, Mapping):
+        return None
+    return cast(Mapping[str, object], container).get(key)  # cast-ok: raw payload, callers re-check every value
+
+
+def _mapping_str_field(container: object, key: str) -> str | None:
+    value: Final = _mapping_field(container, key)
+    return value if isinstance(value, str) and value else None
+
+
+def openai_chat_refusal_text(message_or_delta: object) -> str | None:
+    """
+    Refusal text carried by an OpenAI Chat Completions message or streaming delta,
+    read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it
+    in, or None when the turn is not a refusal.
+    """
+    refusal: Final = getattr(message_or_delta, "refusal", None)
+    if isinstance(refusal, str) and refusal:
+        return refusal
+    return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
+
+
+def _responses_message_refusal_text(item: object) -> str | None:
+    from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
+
+    if isinstance(item, ResponseOutputMessage):
+        return next(
+            (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal),
+            None,
+        )
+    raw_parts: Final = _mapping_field(item, "content")
+    if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence):
+        return None
+    return next(
+        (
+            refusal
+            for part in cast(Sequence[object], raw_parts)  # cast-ok: members re-validated below
+            if _mapping_str_field(part, "type") == "refusal"
+            and isinstance(refusal := _mapping_str_field(part, "refusal"), str)
+        ),
+        None,
+    )
+
+
+def responses_output_refusal_text(output: Iterable[object]) -> str | None:
+    """
+    Refusal text carried by an OpenAI Responses ``output`` list, in typed
+    (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the
+    output messages refused.
+    """
+    return next(
+        (text for item in output if (text := _responses_message_refusal_text(item)) is not None),
+        None,
+    )
+
+
 def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError":
     """The exception a safeguard-refused Anthropic response converts into so the
     content-policy fallback chain can re-dispatch it."""

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -1,13 +1,18 @@
 # What is this?
 ## Translates OpenAI call to Anthropic `/v1/messages` format
+import asyncio
 import json
 import traceback
 from collections import deque
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    refusal_stop_details,
+    responses_output_refusal_text,
+)
 from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
 
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
@@ -49,6 +54,8 @@ class AnthropicResponsesStreamWrapper:
         self._sent_message_start = False
         self._sent_message_stop = False
         self._chunk_queue: deque[dict[str, object]] = deque()
+        self._refusal_text: str = ""
+        self._sync_responses_iterator: Iterator[object] | None = None
 
     def _make_message_start(self) -> dict[str, object]:
         return {
@@ -131,6 +138,24 @@ class AnthropicResponsesStreamWrapper:
                 )
             return
 
+        if event_type == "response.refusal.delta":
+            delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
+            if not isinstance(delta, str) or not delta:
+                return
+            self._refusal_text = self._refusal_text + delta
+            item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
+            block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
+            if block_idx < 0:
+                block_idx = self._open_block(item_id, {"type": "text", "text": ""})
+            self._chunk_queue.append(
+                {
+                    "type": "content_block_delta",
+                    "index": block_idx,
+                    "delta": {"type": "text_delta", "text": delta},
+                }
+            )
+            return
+
         # ---- text delta ----
         if event_type == "response.output_text.delta":
             item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
@@ -215,34 +240,47 @@ class AnthropicResponsesStreamWrapper:
             response_obj: Final = getattr(event, "response", None) or (
                 event.get("response") if isinstance(event, dict) else None
             )
-            stop_reason = "end_turn"
-            anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0)
-
-            if response_obj is not None:
-                status: Final = getattr(response_obj, "status", None)
-                if status == "incomplete":
-                    stop_reason = "max_tokens"
-                anthropic_usage = (
-                    LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage(
-                        getattr(response_obj, "usage", None)
-                    )
+            output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
+            refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
+            status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
+            has_tool_call: Final = any(
+                getattr(item, "type", None) == "function_call"
+                or (isinstance(item, dict) and item.get("type") == "function_call")
+                for item in output
+            )
+            stop_reason: Final = (
+                "max_tokens"
+                if status == "incomplete"
+                else "refusal"
+                if refusal_text is not None
+                else "tool_use"
+                if has_tool_call
+                else "end_turn"
+            )
+            anthropic_usage: Final[AnthropicUsage] = (
+                LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage(
+                    getattr(response_obj, "usage", None)
                 )
+                if response_obj is not None
+                else AnthropicUsage(input_tokens=0, output_tokens=0)
+            )
 
-            # Check if tool_use was in the output to override stop_reason
-            if response_obj is not None:
-                output: Final = getattr(response_obj, "output", []) or []
-                for out_item in output:
-                    out_type = getattr(out_item, "type", None) or (
-                        out_item.get("type") if isinstance(out_item, dict) else None
-                    )
-                    if out_type == "function_call":
-                        stop_reason = "tool_use"
-                        break
+            message_delta_payload: Final = {  # mutable-ok: fresh message_delta payload built per chunk
+                "stop_reason": stop_reason,
+                "stop_sequence": None,
+                **(
+                    {  # mutable-ok: fresh message_delta stop_details entry built per chunk
+                        "stop_details": refusal_stop_details(refusal_text)
+                    }
+                    if stop_reason == "refusal"
+                    else {}  # mutable-ok: empty spread placeholder for non-refusal stop
+                ),
+            }
 
             self._chunk_queue.append(
                 {
                     "type": "message_delta",
-                    "delta": {"stop_reason": stop_reason, "stop_sequence": None},
+                    "delta": message_delta_payload,
                     "usage": dict(anthropic_usage),
                 }
             )
@@ -266,10 +304,20 @@ class AnthropicResponsesStreamWrapper:
 
         # Consume the upstream stream
         try:
-            async for event in self.responses_stream:
-                self._process_event(event)
-                if self._chunk_queue:
-                    return self._chunk_queue.popleft()
+            if hasattr(self.responses_stream, "__aiter__"):
+                async for event in self.responses_stream:
+                    self._process_event(event)
+                    if self._chunk_queue:
+                        return self._chunk_queue.popleft()
+            else:
+                if self._sync_responses_iterator is None:
+                    self._sync_responses_iterator = iter(self.responses_stream)
+                sync_iterator: Final = self._sync_responses_iterator
+                missing: Final = object()
+                while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing:
+                    self._process_event(event)
+                    if self._chunk_queue:
+                        return self._chunk_queue.popleft()
         except StopAsyncIteration:
             pass
         except Exception as e:

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    refusal_stop_details,
+    responses_output_refusal_text,
+)
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     prompt_cache_key_from_user_id,
@@ -624,6 +628,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
         content: Final[list[dict[str, object]]] = []
         stop_reason: AnthropicFinishReason = "end_turn"
+        refusal_text: Final = responses_output_refusal_text(
+            cast(Iterable[object], response.output)  # cast-ok: output items re-validated per item
+        )
 
         for item in response.output:
             if isinstance(item, ResponseReasoningItem):
@@ -631,10 +638,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
             elif isinstance(item, ResponseOutputMessage):
                 for part in item.content:
-                    if getattr(part, "type", None) == "output_text":
+                    part_type = getattr(part, "type", None)
+                    if part_type == "output_text":
                         content.append(
                             AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump()
                         )
+                    elif part_type == "refusal":
+                        content.append(
+                            AnthropicResponseContentBlockText(
+                                type="text", text=getattr(part, "refusal", "") or ""
+                            ).model_dump()
+                        )
 
             elif isinstance(item, ResponseFunctionToolCall):
                 try:
@@ -647,18 +661,28 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         id=item.call_id or item.id or "",
                         name=item.name,
                         input=input_data,
-                    ).model_dump()
+                    ).model_dump(exclude_none=True)
                 )
                 stop_reason = "tool_use"
 
             elif isinstance(item, dict):
                 item_type = item.get("type")
                 if item_type == "message":
-                    for part in item.get("content", []):
-                        if isinstance(part, dict) and part.get("type") == "output_text":
-                            content.append(
-                                AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
-                            )
+                    for part in item.get("content", ()):
+                        if isinstance(part, dict):
+                            part_type = part.get("type")
+                            if part_type == "output_text":
+                                content.append(
+                                    AnthropicResponseContentBlockText(
+                                        type="text", text=part.get("text", "")
+                                    ).model_dump()
+                                )
+                            elif part_type == "refusal":
+                                content.append(
+                                    AnthropicResponseContentBlockText(
+                                        type="text", text=part.get("refusal", "") or ""
+                                    ).model_dump()
+                                )
                 elif item_type == "reasoning":
                     content.extend(
                         self._thinking_blocks_from_reasoning_item(
@@ -676,13 +700,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             id=item.get("call_id") or item.get("id", ""),
                             name=item.get("name", ""),
                             input=input_data,
-                        ).model_dump()
+                        ).model_dump(exclude_none=True)
                     )
                     stop_reason = "tool_use"
-
-        # status -> stop_reason override
         if response.status == "incomplete":
             stop_reason = "max_tokens"
+        elif refusal_text is not None:
+            stop_reason = "refusal"
 
         anthropic_usage: Final = self.translate_responses_api_usage_to_anthropic_usage(response.usage)
 
@@ -695,4 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             usage=anthropic_usage,
             content=content,
             stop_reason=stop_reason,
+            stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None),
         )

litellm/llms/azure_ai/chat/transformation.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
+from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
 from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
 from litellm.llms.openai.openai import OpenAIConfig
 from litellm.llms.xai.chat.transformation import XAIChatConfig
@@ -42,12 +43,37 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
 )
 
 
+class AzureAIGPT5Config(OpenAIGPT5Config):
+    @classmethod
+    def _model_map_lookup_name(cls, model: str) -> str:
+        """Normalise a Foundry routing name to its cost-map key, when the map has one.
+
+        A Foundry deployment and its OpenAI-hosted namesake are different products with
+        different capabilities, so ``azure_ai/<model>`` is the entry to read whenever the map
+        carries it. Most gpt-5-family names have no ``azure_ai/`` row, though, and prefixing
+        those anyway costs them every flag: ``get_llm_provider`` re-resolves an ``azure_ai/``
+        name to the azure provider when a global AZURE_AI_API_BASE points at an
+        openai.azure.com host, ``azure/<model>`` is not a key either, so the lookup lands
+        nowhere and every effort answer degrades to False. A missing key defers to the base
+        resolver instead.
+        """
+        prefixed: Final = model if model.startswith("azure_ai/") else f"azure_ai/{model}"
+        return prefixed if prefixed in litellm.model_cost else super()._model_map_lookup_name(model)
+
+
+azureAIGPT5Config: Final = AzureAIGPT5Config()
+
+
 class AzureAIStudioConfig(OpenAIConfig):
     def get_supported_openai_params(self, model: str) -> list:
         model_supports_tool_choice = True  # azure ai supports this by default
         if not supports_tool_choice(model=f"azure_ai/{model}"):
             model_supports_tool_choice = False
-        supported_params = super().get_supported_openai_params(model)
+        supported_params = (
+            azureAIGPT5Config.get_supported_openai_params(model)
+            if azureAIGPT5Config.is_model_gpt_5_model(model)
+            else super().get_supported_openai_params(model)
+        )
         if not model_supports_tool_choice:
             filtered_supported_params: Final = []
             for param in supported_params:
@@ -61,6 +87,27 @@ class AzureAIStudioConfig(OpenAIConfig):
 
         return supported_params
 
+    def map_openai_params(
+        self,
+        non_default_params: dict[str, object],  # mutable-ok: OpenAIConfig.map_openai_params signature
+        optional_params: dict[str, object],  # mutable-ok: OpenAIConfig.map_openai_params signature
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: OpenAIConfig.map_openai_params signature
+        if not azureAIGPT5Config.is_model_gpt_5_model(model):
+            return super().map_openai_params(
+                non_default_params=non_default_params,
+                optional_params=optional_params,
+                model=model,
+                drop_params=drop_params,
+            )
+        return azureAIGPT5Config.map_openai_params(
+            non_default_params=non_default_params,
+            optional_params=optional_params,
+            model=model,
+            drop_params=drop_params,
+        )
+
     def _supports_stop_reason(self, model: str) -> bool:
         """
         Check if the model supports stop tokens.

litellm/llms/azure_ai/ocr/__init__.py

@@ -1,5 +1,6 @@
 """Azure AI OCR module."""
 
+from .cohere_parse_transformation import AzureAICohereParseConfig
 from .common_utils import get_azure_ai_ocr_config
 from .document_intelligence.transformation import (
     AzureDocumentIntelligenceOCRConfig,
@@ -7,6 +8,7 @@ from .document_intelligence.transformation import (
 from .transformation import AzureAIOCRConfig
 
 __all__ = [
+    "AzureAICohereParseConfig",
     "AzureAIOCRConfig",
     "AzureDocumentIntelligenceOCRConfig",
     "get_azure_ai_ocr_config",

litellm/llms/azure_ai/ocr/cohere_parse_transformation.py

@@ -0,0 +1,91 @@
+"""Cohere Parse served from Azure AI Foundry (`/providers/cohere/v2/parse`)."""
+
+from collections.abc import Mapping
+from typing import Final
+
+import httpx
+
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    async_convert_url_to_base64,
+    convert_url_to_base64,
+)
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
+from litellm.llms.cohere.ocr.transformation import COHERE_PARSE_PATH, CohereParseConfig
+from litellm.secret_managers.main import get_secret_str
+
+AZURE_AI_API_KEY_ENV_VAR: Final = "AZURE_AI_API_KEY"
+AZURE_AI_API_BASE_ENV_VAR: Final = "AZURE_AI_API_BASE"
+AZURE_AI_COHERE_PROVIDER_PATH: Final = "/providers/cohere"
+AZURE_AI_MODELS_PATH_SUFFIX: Final = "/models"
+
+
+class AzureAICohereParseConfig(CohereParseConfig):
+    """Same request and response shape as Cohere Parse, behind Azure AI auth and URL layout.
+
+    Foundry cannot fetch external URLs, so remote images are inlined as base64 data URIs.
+    """
+
+    def get_api_key_env_var(self) -> str | None:
+        return AZURE_AI_API_KEY_ENV_VAR
+
+    def _llm_provider(self) -> str:
+        return "azure_ai"
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: BaseOCRConfig signature
+        resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
+        if resolved_base is None:
+            raise ValueError(
+                f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
+                "or pass api_base parameter"
+            )
+        resolved_key: Final = api_key or get_secret_str(AZURE_AI_API_KEY_ENV_VAR)
+        return {  # mutable-ok: BaseOCRConfig signature
+            **get_azure_ai_auth_headers(api_key=resolved_key, litellm_params=litellm_params),
+            "Content-Type": "application/json",
+            **headers,
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object] | None = None,
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.get_complete_url signature
+    ) -> str:
+        resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
+        if resolved_base is None:
+            raise ValueError(
+                f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
+                "or pass api_base parameter"
+            )
+        url: Final = httpx.URL(resolved_base)
+        if not url.is_absolute_url:
+            raise ValueError(
+                "Azure AI API Base must be an absolute URL including scheme (e.g. "
+                f"'https://<resource>.services.ai.azure.com'). Got api_base={resolved_base!r}."
+            )
+        path: Final = url.path.rstrip("/")
+        if path.endswith(COHERE_PARSE_PATH):
+            return str(url.copy_with(path=path))
+        if path.endswith(f"{AZURE_AI_COHERE_PROVIDER_PATH}/v2"):
+            return str(url.copy_with(path=f"{path}/parse"))
+        return str(
+            url.copy_with(
+                path=f"{path.removesuffix(AZURE_AI_MODELS_PATH_SUFFIX)}{AZURE_AI_COHERE_PROVIDER_PATH}{COHERE_PARSE_PATH}"
+            )
+        )
+
+    def _resolve_image_url_sync(self, image_url: str) -> str:
+        return convert_url_to_base64(image_url)
+
+    async def _resolve_image_url_async(self, image_url: str) -> str:
+        return await async_convert_url_to_base64(image_url)

litellm/llms/azure_ai/ocr/common_utils.py

@@ -24,6 +24,11 @@ def is_azure_document_intelligence_model(model: str) -> bool:
     return "doc-intelligence" in lowered or "documentintelligence" in lowered
 
 
+def is_azure_cohere_parse_model(model: str) -> bool:
+    lowered: Final = model.lower()
+    return "cohere" in lowered and "parse" in lowered
+
+
 def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
     """
     Determine which Azure AI OCR configuration to use based on the model name.
@@ -46,6 +51,7 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
         >>> get_azure_ai_ocr_config("azure_ai/pixtral-12b-2409")
         <AzureAIOCRConfig object>
     """
+    from litellm.llms.azure_ai.ocr.cohere_parse_transformation import AzureAICohereParseConfig
     from litellm.llms.azure_ai.ocr.document_intelligence.transformation import (
         AzureDocumentIntelligenceOCRConfig,
     )
@@ -56,6 +62,10 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
         verbose_logger.debug("Routing %s to Azure Document Intelligence OCR config", model)
         return AzureDocumentIntelligenceOCRConfig()
 
+    if is_azure_cohere_parse_model(model):
+        verbose_logger.debug("Routing %s to Azure AI Cohere Parse config", model)
+        return AzureAICohereParseConfig()
+
     # Default to Mistral-based OCR for other azure_ai models
     verbose_logger.debug("Routing %s to Azure AI (Mistral) OCR config", model)
     return AzureAIOCRConfig()

litellm/llms/base_llm/ocr/transformation.py

@@ -33,6 +33,8 @@ OCR_REQUEST_FORMAT_HEADER: Final = "x-req-format"
 
 PROVIDER_NATIVE_RESPONSE_KEY: Final = "provider_native_response"
 
+HEALTH_CHECK_PDF_DATA_URI: Final = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
+
 
 def parse_ocr_request_format(value: object) -> OCRRequestFormat:
     if value == "litellm":
@@ -142,6 +144,16 @@ class BaseOCRConfig:
         """
         return None
 
+    def supports_rust_bridge(self) -> bool:
+        """Whether the Rust OCR bridge may serve this config when it is enabled for the provider."""
+        return True
+
+    def get_health_check_document(self) -> DocumentType:
+        return {  # mutable-ok: litellm.aocr rejects any document that is not a dict
+            "type": "document_url",
+            "document_url": HEALTH_CHECK_PDF_DATA_URI,
+        }
+
     def map_ocr_params(
         self,
         non_default_params: dict,

litellm/llms/cohere/ocr/__init__.py

@@ -0,0 +1,3 @@
+from litellm.llms.cohere.ocr.transformation import CohereParseConfig
+
+__all__ = ("CohereParseConfig",)

litellm/llms/cohere/ocr/transformation.py

@@ -0,0 +1,301 @@
+"""Cohere Parse (`POST /v2/parse`) exposed through LiteLLM's OCR interface."""
+
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+import httpx
+from pydantic import BaseModel, ConfigDict, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.exceptions import BadRequestError, UnsupportedParamsError
+from litellm.llms.base_llm.ocr.transformation import (
+    OCR_REQUEST_FORMAT_PARAM,
+    BaseOCRConfig,
+    DocumentType,
+    OCRPage,
+    OCRPageImage,
+    OCRRequestData,
+    OCRRequestFormat,
+    OCRResponse,
+    OCRUsageInfo,
+    parse_ocr_request_format,
+)
+from litellm.llms.cohere.common_utils import CohereError
+from litellm.secret_managers.main import get_secret_str
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+COHERE_API_KEY_ENV_VAR: Final = "COHERE_API_KEY"
+COHERE_PARSE_API_BASE: Final = "https://api.cohere.com"
+COHERE_PARSE_PATH: Final = "/v2/parse"
+COHERE_PARSE_OUTPUT_FORMAT_PARAM: Final = "output_format"
+COHERE_PARSE_OUTPUT_FORMATS: Final = ("markdown", "blocks")
+COHERE_PARSE_DEFAULT_OUTPUT_FORMAT: Final = "markdown"
+COHERE_PARSE_SUPPORTED_PARAMS: Final = (COHERE_PARSE_OUTPUT_FORMAT_PARAM, OCR_REQUEST_FORMAT_PARAM)
+COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: Final = (
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
+)
+COHERE_PARSE_IMAGE_ONLY_MESSAGE: Final = (
+    "Cohere Parse only accepts `image_url` documents (an image URL or a base64 image data URI); "
+    "`document_url` and PDF inputs are not supported."
+)
+
+_NATIVE_RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object])
+_BOUNDING_BOX_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+class _CohereParseDocument(TypedDict):
+    type: ReadOnly[Literal["image_url"]]
+    image_url: ReadOnly[str]
+
+
+class _CohereParseRequestBody(TypedDict):
+    model: ReadOnly[str]
+    document: ReadOnly[_CohereParseDocument]
+    output_format: ReadOnly[str]
+
+
+class _MarkdownPage(TypedDict):
+    index: ReadOnly[int]
+    markdown: ReadOnly[str]
+    images: ReadOnly[Sequence[OCRPageImage] | None]
+
+
+class _BlocksPage(_MarkdownPage):
+    blocks: ReadOnly[Sequence[Mapping[str, object]]]
+
+
+class _CohereParseMarkdown(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    content: str = ""
+    images: Sequence[Mapping[str, object]] | None = None
+
+
+class _CohereParsePage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    index: int | None = None
+    markdown: _CohereParseMarkdown | None = None
+    blocks: Sequence[Mapping[str, object]] | None = None
+
+
+class _CohereParseBilledUnits(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    pages: int | None = None
+
+
+class _CohereParseMeta(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    billed_units: _CohereParseBilledUnits | None = None
+
+
+class _CohereParseResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    pages: Sequence[_CohereParsePage] = ()
+    meta: _CohereParseMeta | None = None
+
+
+def _requested_format(optional_params: Mapping[str, object] | None) -> OCRRequestFormat:
+    if optional_params is None:
+        return "litellm"
+    return "native" if optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native" else "litellm"
+
+
+def _page_image(image: Mapping[str, object]) -> OCRPageImage:
+    bounding_box: Final = image.get("bounding_box")
+    if not isinstance(bounding_box, Mapping):
+        return OCRPageImage.model_validate(image)
+    bbox: Final = _BOUNDING_BOX_ADAPTER.validate_python(bounding_box)
+    return OCRPageImage.model_validate(MappingProxyType({**image, "bbox": bbox}))
+
+
+def _normalize_page(page: _CohereParsePage, position: int) -> OCRPage:
+    markdown: Final = page.markdown
+    images: Final = tuple(_page_image(image) for image in markdown.images) if markdown and markdown.images else None
+    normalized: Final[_MarkdownPage] = {
+        "index": page.index if page.index is not None else position,
+        "markdown": markdown.content if markdown else "",
+        "images": images,
+    }
+    if page.blocks is None:
+        return OCRPage.model_validate(normalized)
+    with_blocks: Final[_BlocksPage] = {**normalized, "blocks": page.blocks}
+    return OCRPage.model_validate(with_blocks)
+
+
+def _billed_pages(parsed: _CohereParseResponse) -> int | None:
+    if parsed.meta is None or parsed.meta.billed_units is None:
+        return None
+    return parsed.meta.billed_units.pages
+
+
+class CohereParseConfig(BaseOCRConfig):
+    """Cohere Parse, an image-only document understanding endpoint returning markdown or blocks."""
+
+    def get_supported_ocr_params(self, model: str) -> list[str]:  # mutable-ok: BaseOCRConfig signature
+        return list(COHERE_PARSE_SUPPORTED_PARAMS)  # mutable-ok: BaseOCRConfig signature
+
+    def get_api_key_env_var(self) -> str | None:
+        return COHERE_API_KEY_ENV_VAR
+
+    def supports_rust_bridge(self) -> bool:
+        return False
+
+    def get_health_check_document(self) -> DocumentType:
+        return {  # mutable-ok: litellm.aocr rejects any document that is not a dict
+            "type": "image_url",
+            "image_url": COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI,
+        }
+
+    def _llm_provider(self) -> str:
+        return "cohere"
+
+    def map_ocr_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+    ) -> dict[str, object]:  # mutable-ok: BaseOCRConfig signature
+        output_format: Final = non_default_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM)
+        if output_format is not None and output_format not in COHERE_PARSE_OUTPUT_FORMATS:
+            raise UnsupportedParamsError(
+                message=(
+                    f"Invalid `{COHERE_PARSE_OUTPUT_FORMAT_PARAM}`: {output_format!r}. "
+                    f"Expected one of {', '.join(COHERE_PARSE_OUTPUT_FORMATS)}."
+                ),
+                model=model,
+                llm_provider=self._llm_provider(),
+            )
+        requested_format: Final = non_default_params.get(OCR_REQUEST_FORMAT_PARAM)
+        request_format: Final = parse_ocr_request_format(requested_format) if requested_format is not None else None
+        overrides: Final = tuple(
+            (key, value)
+            for key, value in (
+                (COHERE_PARSE_OUTPUT_FORMAT_PARAM, output_format),
+                (OCR_REQUEST_FORMAT_PARAM, request_format),
+            )
+            if value is not None
+        )
+        return {**optional_params, **dict(overrides)}  # mutable-ok: BaseOCRConfig signature
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: BaseOCRConfig signature
+        resolved_key: Final = api_key or get_secret_str(COHERE_API_KEY_ENV_VAR)
+        if resolved_key is None:
+            raise ValueError(
+                f"Missing {COHERE_API_KEY_ENV_VAR} - set it in the environment or pass api_key to "
+                "litellm.ocr()/litellm.aocr()"
+            )
+        return {  # mutable-ok: BaseOCRConfig signature
+            "Authorization": f"Bearer {resolved_key}",
+            "Content-Type": "application/json",
+            **headers,
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object] | None = None,
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.get_complete_url signature
+    ) -> str:
+        url: Final = httpx.URL(api_base or COHERE_PARSE_API_BASE)
+        path: Final = url.path.rstrip("/")
+        if path.endswith(COHERE_PARSE_PATH):
+            return str(url.copy_with(path=path))
+        if path.endswith("/v2"):
+            return str(url.copy_with(path=f"{path}/parse"))
+        return str(url.copy_with(path=f"{path}{COHERE_PARSE_PATH}"))
+
+    def _image_url(self, document: DocumentType, model: str) -> str:
+        image_url: Final = document.get("image_url", "")
+        if document.get("type") != "image_url" or not image_url or image_url.startswith("data:application/pdf"):
+            raise BadRequestError(
+                message=COHERE_PARSE_IMAGE_ONLY_MESSAGE,
+                model=model,
+                llm_provider=self._llm_provider(),
+            )
+        return image_url
+
+    def _resolve_image_url_sync(self, image_url: str) -> str:
+        return image_url
+
+    async def _resolve_image_url_async(self, image_url: str) -> str:
+        return image_url
+
+    def _build_request(self, model: str, image_url: str, optional_params: Mapping[str, object]) -> OCRRequestData:
+        body: Final[_CohereParseRequestBody] = {
+            "model": model,
+            "document": {"type": "image_url", "image_url": image_url},
+            "output_format": str(
+                optional_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM, COHERE_PARSE_DEFAULT_OUTPUT_FORMAT)
+            ),
+        }
+        return OCRRequestData(data=dict(body), files=None)  # mutable-ok: OCRRequestData.data is a dict
+
+    def transform_ocr_request(
+        self,
+        model: str,
+        document: DocumentType,
+        optional_params: Mapping[str, object],
+        headers: Mapping[str, str],
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.transform_ocr_request signature
+    ) -> OCRRequestData:
+        image_url: Final = self._resolve_image_url_sync(self._image_url(document, model))
+        return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
+
+    async def async_transform_ocr_request(
+        self,
+        model: str,
+        document: DocumentType,
+        optional_params: Mapping[str, object],
+        headers: Mapping[str, str],
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.async_transform_ocr_request signature
+    ) -> OCRRequestData:
+        image_url: Final = await self._resolve_image_url_async(self._image_url(document, model))
+        return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
+
+    def transform_ocr_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+        optional_params: Mapping[str, object] | None = None,
+        **kwargs: object,  # kwargs-ok: BaseOCRConfig.transform_ocr_response signature
+    ) -> OCRResponse:
+        native: Final = _NATIVE_RESPONSE_ADAPTER.validate_python(raw_response.json())
+        parsed: Final = _CohereParseResponse.model_validate(native)
+        pages: Final = [  # mutable-ok: OCRResponse.pages is a list
+            _normalize_page(page, position) for position, page in enumerate(parsed.pages)
+        ]
+        billed_pages: Final = _billed_pages(parsed)
+        response: Final = OCRResponse(
+            pages=pages,
+            model=model,
+            usage_info=OCRUsageInfo(pages_processed=billed_pages if billed_pages is not None else len(pages)),
+        )
+        if _requested_format(optional_params) == "native":
+            response.set_provider_native_response(native)
+        return response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: Mapping[str, str],
+    ) -> Exception:
+        return CohereError(status_code=status_code, message=error_message)

litellm/llms/vertex_ai/common_utils.py

@@ -1,9 +1,14 @@
 import re
+from collections.abc import Mapping
 from copy import deepcopy
 from enum import Enum
+from functools import lru_cache
+from types import MappingProxyType
 from typing import Any, Final, Literal, cast, get_type_hints
 
 import httpx
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -21,6 +26,61 @@ from litellm.types.utils import TokenCountResponse
 from litellm.utils import supports_response_schema, supports_system_messages
 
 
+class VertexAILyriaModelInfo(TypedDict):
+    vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]]
+    supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]]
+    output_cost_per_image: NotRequired[ReadOnly[float]]
+
+
+_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo)
+
+
+def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyriaModelInfo | None:
+    if raw_model_info is None:
+        return None
+    try:
+        return _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER.validate_python(raw_model_info)
+    except ValidationError:
+        return None
+
+
+@lru_cache(maxsize=1)
+def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]:
+    from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
+
+    return MappingProxyType(
+        {
+            model_key: lyria_model_info
+            for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items()
+            if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None
+        }
+    )
+
+
+def _vertex_ai_lyria_model_key(model: str) -> str:
+    return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
+
+
+def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None:
+    return None if model_info is None else model_info.get("output_cost_per_image")
+
+
+def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None:
+    model_key: Final = _vertex_ai_lyria_model_key(model)
+    runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
+    return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key)
+
+
+def get_vertex_ai_lyria_generation_cost(model: str) -> float | None:
+    model_key: Final = _vertex_ai_lyria_model_key(model)
+    runtime_cost: Final = _vertex_ai_lyria_generation_cost(
+        _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
+    )
+    if runtime_cost is not None:
+        return runtime_cost
+    return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key))
+
+
 class VertexAIError(BaseLLMException):
     def __init__(
         self,

litellm/llms/vertex_ai/text_to_speech/transformation.py

@@ -8,17 +8,25 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s
 import base64
 from collections.abc import Coroutine
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union
 
 import httpx
 
+import litellm
+from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.audio_utils.utils import (
+    DEFAULT_SPEECH_MEDIA_TYPE,
     speech_media_type_from_audio_bytes,
 )
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.text_to_speech.transformation import (
     BaseTextToSpeechConfig,
     TextToSpeechRequestData,
 )
+from litellm.llms.vertex_ai.common_utils import (
+    VertexAILyriaModelInfo,
+    get_vertex_ai_lyria_model_info,
+)
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from litellm.types.llms.vertex_ai_text_to_speech import (
@@ -35,6 +43,10 @@ else:
     LiteLLMLoggingObj = Any
     HttpxBinaryResponseContent = Any
 
+_LyriaVoice: TypeAlias = (
+    str | dict | None
+)  # mutable-ok: inherited interface supports structured provider voice dictionaries
+
 
 class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
     """
@@ -472,3 +484,209 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
 
         # Initialize the HttpxBinaryResponseContent instance
         return HttpxBinaryResponseContent(response)
+
+
+class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig):
+    @classmethod
+    def is_lyria_model(cls, model: str) -> bool:
+        return get_vertex_ai_lyria_model_info(model=model) is not None
+
+    @staticmethod
+    def _get_model_info(model: str) -> VertexAILyriaModelInfo:
+        model_info: Final = get_vertex_ai_lyria_model_info(model=model)
+        if model_info is None:
+            raise ValueError(f"Vertex AI model {model!r} does not declare a Lyria audio API")
+        return model_info
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list:  # mutable-ok: inherited provider interface returns a concrete parameter list
+        return [  # mutable-ok: inherited provider interface requires a concrete parameter list
+            "response_format"
+        ]
+
+    def map_openai_params(
+        self,
+        model: str,
+        optional_params: dict,  # mutable-ok: inherited provider interface accepts a concrete parameter dictionary
+        voice: _LyriaVoice = None,
+        drop_params: bool = False,
+        kwargs: dict | None = None,  # mutable-ok: inherited provider interface accepts a concrete keyword dictionary
+    ) -> tuple[str | None, dict]:  # mutable-ok: inherited provider interface returns concrete mapped parameters
+        mapped_params: Final = dict(  # mutable-ok: mapping drops unsupported parameters before provider dispatch
+            optional_params
+        )
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        unsupported_params: Final = tuple(
+            param for param in ("speed", "instructions") if mapped_params.get(param) is not None
+        )
+        if unsupported_params:
+            if drop_params or litellm.drop_params:
+                for param in unsupported_params:
+                    mapped_params.pop(param, None)
+            else:
+                raise UnsupportedParamsError(
+                    status_code=400,
+                    message=(
+                        f"Vertex AI {base_model} does not support the OpenAI parameters: "
+                        f"{', '.join(unsupported_params)}. To drop unsupported openai params "
+                        "from the call, set `litellm.drop_params = True`"
+                    ),
+                )
+        response_format: Final = mapped_params.get("response_format")
+        supported_formats: Final = frozenset(model_info["supported_audio_formats"])
+        if response_format is not None and response_format not in supported_formats:
+            if drop_params or litellm.drop_params:
+                mapped_params.pop("response_format", None)
+            else:
+                raise UnsupportedParamsError(
+                    status_code=400,
+                    message=(
+                        f"Vertex AI {base_model} does not support response_format={response_format!r}. "
+                        f"Supported values: {', '.join(sorted(supported_formats))}. "
+                        "To drop unsupported openai params from the call, set `litellm.drop_params = True`"
+                    ),
+                )
+        return voice if isinstance(voice, str) else None, mapped_params
+
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: dict,  # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters
+    ) -> str:
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        configured_project: Final = self.safe_get_vertex_ai_project(litellm_params)
+        project: Final = (
+            self._ensure_access_token(
+                credentials=self.safe_get_vertex_ai_credentials(litellm_params),
+                project_id=None,
+                custom_llm_provider="vertex_ai",
+            )[1]
+            if configured_project is None
+            else configured_project
+        )
+        if model_info["vertex_ai_audio_api"] == "lyria_interactions":
+            from litellm.llms.vertex_ai.interactions.transformation import (
+                VertexAIInteractionsConfig,
+            )
+
+            def mint_access_token(
+                _credentials: VERTEX_CREDENTIALS_TYPES | None,
+                project_id: str | None,
+            ) -> tuple[str, str]:
+                return "", project_id or project
+
+            return VertexAIInteractionsConfig(mint_access_token=mint_access_token).get_complete_url(
+                api_base=api_base,
+                model=base_model,
+                litellm_params={  # mutable-ok: interactions dispatch expects a concrete parameter dictionary
+                    **litellm_params,
+                    "vertex_project": project,
+                    "vertex_location": "global",
+                },
+            )
+        location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location()
+        base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/")
+        encoded_project: Final = encode_url_path_segment(project, field_name="project")
+        encoded_location: Final = encode_url_path_segment(location, field_name="location")
+        encoded_model: Final = encode_url_path_segment(base_model, field_name="model")
+        return (
+            f"{base_url}/v1/projects/{encoded_project}/locations/{encoded_location}"
+            f"/publishers/google/models/{encoded_model}:predict"
+        )
+
+    def transform_text_to_speech_request(
+        self,
+        model: str,
+        input: str,
+        voice: str | None,
+        optional_params: dict,  # mutable-ok: inherited provider interface accepts concrete mapped parameters
+        litellm_params: dict,  # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters
+        headers: dict,  # mutable-ok: inherited provider interface accepts and updates concrete HTTP headers
+    ) -> TextToSpeechRequestData:
+        access_token, project = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(litellm_params),
+            project_id=self.safe_get_vertex_ai_project(litellm_params),
+            custom_llm_provider="vertex_ai",
+        )
+        headers.update(
+            {  # mutable-ok: HTTP dispatch requires a concrete header dictionary
+                "Authorization": f"Bearer {access_token}",
+                "x-goog-user-project": project,
+                "Content-Type": "application/json",
+            }
+        )
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        request_body: Final[dict[str, object]] = (  # mutable-ok: HTTP dispatch requires a concrete provider payload
+            {  # mutable-ok: predict dispatch requires a concrete provider request dictionary
+                "instances": [  # mutable-ok: predict dispatch requires a concrete instances list
+                    {"prompt": input}  # mutable-ok: predict dispatch requires a concrete instance dictionary
+                ],
+                "parameters": {  # mutable-ok: predict dispatch requires a concrete parameters dictionary
+                    "sample_count": 1
+                },
+            }
+            if model_info["vertex_ai_audio_api"] == "lyria_predict"
+            else {  # mutable-ok: interactions dispatch requires a concrete provider request dictionary
+                "model": base_model,
+                "input": input,
+                **(
+                    {  # mutable-ok: interactions dispatch requires a nested response-format dictionary
+                        "response_format": {  # mutable-ok: interactions response format is a concrete provider payload
+                            "type": "audio",
+                            "mime_type": "audio/wav",
+                        }
+                    }
+                    if optional_params.get("response_format") == "wav"
+                    else {}  # mutable-ok: no response override is merged for non-WAV output
+                ),
+            }
+        )
+        return TextToSpeechRequestData(dict_body=request_body, headers=headers)
+
+    def transform_text_to_speech_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> "HttpxBinaryResponseContent":
+        from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+        response_json: Final = raw_response.json()
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        audio_data: str | None = None  # rebind-ok: response parsing discovers audio data in provider-specific shapes
+        mime_type: str | None = None  # rebind-ok: response parsing discovers the MIME type beside the audio payload
+        if model_info["vertex_ai_audio_api"] == "lyria_predict":
+            predictions: Final = response_json.get("predictions") or ()
+            if predictions:
+                audio_data = predictions[0].get("audioContent") or predictions[0].get(
+                    "bytesBase64Encoded"
+                )  # rebind-ok: predict response supplies the generated audio value
+                mime_type = predictions[0].get("mimeType")  # rebind-ok: predict response supplies its audio MIME type
+        else:
+            for step in response_json.get("steps") or response_json.get("outputs") or ():
+                content_items = step.get("content") or () if step.get("type") == "model_output" else (step,)
+                for content in content_items:
+                    if content.get("type") == "audio" and content.get("data"):
+                        audio_data = content[
+                            "data"
+                        ]  # rebind-ok: interactions response supplies the generated audio value
+                        mime_type = content.get(
+                            "mime_type"
+                        )  # rebind-ok: interactions response supplies its audio MIME type
+        if audio_data is None:
+            raise ValueError(f"No generated audio found in Vertex AI {base_model} response")
+        binary_data: Final = base64.b64decode(audio_data)
+        media_type: Final = mime_type or speech_media_type_from_audio_bytes(binary_data) or DEFAULT_SPEECH_MEDIA_TYPE
+        return HttpxBinaryResponseContent(
+            httpx.Response(
+                status_code=raw_response.status_code,
+                content=binary_data,
+                headers=MappingProxyType({"content-type": media_type}),
+            )
+        )

litellm/main.py

@@ -8247,6 +8247,7 @@ def speech(
         )
     elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
         from litellm.llms.vertex_ai.text_to_speech.transformation import (
+            VertexAILyriaTextToSpeechConfig,
             VertexAITextToSpeechConfig,
         )
 
@@ -8271,7 +8272,11 @@ def speech(
 
         # Vertex AI Text-to-Speech (Google Cloud TTS)
         if text_to_speech_provider_config is None:
-            text_to_speech_provider_config = VertexAITextToSpeechConfig()
+            text_to_speech_provider_config = (  # rebind-ok: model metadata selects the Vertex TTS implementation
+                VertexAILyriaTextToSpeechConfig()
+                if VertexAILyriaTextToSpeechConfig.is_lyria_model(model)
+                else VertexAITextToSpeechConfig()
+            )
 
         # Cast to specific Vertex AI config type to access dispatch method
         vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config)

litellm/model_prices_and_context_window_backup.json

@@ -3485,6 +3485,55 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "azure_ai/gpt-6-astra": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-06,
+        "input_cost_per_token": 1e-05,
+        "input_cost_per_token_above_272k_tokens": 2e-05,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "output_cost_per_token_above_272k_tokens": 7.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "source": "https://ai.azure.com/catalog/models/gpt-6-astra",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": false,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "azure_ai/gpt-5.5": {
         "deprecation_date": "2027-10-26",
         "cache_read_input_token_cost": 5e-07,
@@ -7189,7 +7238,7 @@
         ],
         "supports_computer_use": true,
         "supports_function_calling": true,
-        "supports_max_reasoning_effort": true,
+        "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
         "supports_none_reasoning_effort": true,
@@ -7455,7 +7504,7 @@
         ],
         "supports_computer_use": true,
         "supports_function_calling": true,
-        "supports_max_reasoning_effort": true,
+        "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
         "supports_none_reasoning_effort": true,
@@ -10243,6 +10292,16 @@
         ],
         "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/"
     },
+    "azure_ai/Cohere-parse-v5": {
+        "deprecation_date": "2026-12-15",
+        "litellm_provider": "azure_ai",
+        "mode": "ocr",
+        "ocr_cost_per_page": 0.0015,
+        "source": "https://cohere.com/blog/parse",
+        "supported_endpoints": [
+            "/v1/ocr"
+        ]
+    },
     "azure_ai/doc-intelligence/prebuilt-read": {
         "litellm_provider": "azure_ai",
         "ocr_cost_per_page": 0.0015,
@@ -12228,6 +12287,48 @@
         "output_cost_per_token": 2.65e-06,
         "supports_pdf_input": true
     },
+    "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": {
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 8172,
+        "max_tokens": 8172,
+        "mode": "embedding",
+        "input_cost_per_token": 1.62e-07,
+        "input_cost_per_image": 7.2e-05,
+        "input_cost_per_video_per_second": 0.00084,
+        "input_cost_per_audio_per_second": 0.000168,
+        "output_cost_per_token": 0.0,
+        "output_vector_size": 3072,
+        "supports_embedding_image_input": true,
+        "supports_image_input": true,
+        "supports_video_input": true,
+        "supports_audio_input": true
+    },
+    "bedrock/us-gov-west-1/amazon.nova-lite-v1:0": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 300000,
+        "max_output_tokens": 10000,
+        "max_tokens": 10000,
+        "mode": "chat",
+        "output_cost_per_token": 2.88e-07,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 10000,
+        "max_tokens": 10000,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-07,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true
+    },
     "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": {
         "input_cost_per_token": 9.6e-07,
         "litellm_provider": "bedrock",
@@ -14116,6 +14217,15 @@
         "output_vector_size": 1536,
         "supports_embedding_image_input": true
     },
+    "cohere/parse-v5.0": {
+        "litellm_provider": "cohere",
+        "mode": "ocr",
+        "ocr_cost_per_page": 0.0015,
+        "source": "https://cohere.com/blog/parse",
+        "supported_endpoints": [
+            "/v1/ocr"
+        ]
+    },
     "cohere.rerank-v3-5:0": {
         "input_cost_per_query": 0.002,
         "input_cost_per_token": 0.0,
@@ -43693,6 +43803,23 @@
         "input_cost_per_token_batches": 1.65e-06,
         "output_cost_per_token_batches": 8.25e-06
     },
+    "us-gov.anthropic.claude-3-haiku-20240307-v1:0": {
+        "deprecation_date": "2026-09-10",
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "cache_read_input_token_cost": 3e-08,
+        "cache_creation_input_token_cost": 3.75e-07
+    },
     "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 4.5e-06,
         "cache_creation_input_token_cost_above_1hr": 7.2e-06,
@@ -43787,6 +43914,160 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "us-gov.anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "us-gov.anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
+    "us-gov.nvidia.nemotron-nano-3-30b": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.88e-07,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_function_calling": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.nvidia.nemotron-nano-12b-v2": {
+        "input_cost_per_token": 2.4e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 7.2e-07,
+        "supports_system_messages": true,
+        "supports_vision": true
+    },
+    "us-gov.nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
+    "us-gov.nvidia.nemotron-super-3-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 7.8e-07,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.openai.gpt-oss-20b-1:0": {
+        "input_cost_per_token": 8.4e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.openai.gpt-oss-120b-1:0": {
+        "input_cost_per_token": 1.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 7.2e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.xai.grok-4.6": {
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "au.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.375e-06,
         "cache_creation_input_token_cost_above_1hr": 2.2e-06,
@@ -46973,6 +47254,99 @@
         "output_cost_per_token": 4e-07,
         "supports_tool_choice": true
     },
+    "vertex_ai/lyria-002": {
+        "litellm_provider": "vertex_ai",
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.06,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supports_audio_output": true,
+        "vertex_ai_audio_api": "lyria_predict"
+    },
+    "vertex_ai/lyria-3-clip-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.04,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
+    "vertex_ai/lyria-3-pro-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.08,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3",
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
     "vertex_ai/meta/llama-3.1-405b-instruct-maas": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "vertex_ai-llama_models",
@@ -59608,6 +59982,16 @@
         "supports_system_messages": true,
         "supports_vision": true
     },
+    "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
     "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": {
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "bedrock",
@@ -59712,6 +60096,70 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "bedrock/us-gov-west-1/anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
     "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": {
         "input_cost_per_token": 7.2e-08,
         "litellm_provider": "bedrock",
@@ -59737,6 +60185,16 @@
         "supports_system_messages": true,
         "supports_vision": true
     },
+    "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
     "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": {
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "bedrock",
@@ -59841,6 +60299,70 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "bedrock/us-gov-east-1/anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": {
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
@@ -59955,6 +60477,120 @@
         "output_cost_per_token": 3e-06,
         "cache_read_input_token_cost": 2.4e-07
     },
+    "bedrock_mantle/us-gov-west-1/xai.grok-4.6": {
+        "use_openai_responses_path": true,
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": {
+        "input_cost_per_token": 4.8e-08,
+        "output_cost_per_token": 9.6e-08,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": {
+        "input_cost_per_token": 1.56e-07,
+        "output_cost_per_token": 4.8e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-31b": {
+        "input_cost_per_token": 1.68e-07,
+        "output_cost_per_token": 4.8e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
+        "input_cost_per_token": 8.4e-08,
+        "output_cost_per_token": 3.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "output_cost_per_token": 7.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": {
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
@@ -59982,6 +60618,63 @@
         "cache_read_input_token_cost": 3.3e-07,
         "output_cost_per_token": 1.98e-05
     },
+    "bedrock_mantle/us-gov-east-1/xai.grok-4.6": {
+        "use_openai_responses_path": true,
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
+        "input_cost_per_token": 8.4e-08,
+        "output_cost_per_token": 3.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "output_cost_per_token": 7.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "azure/us-gov/gpt-5.1": {
         "cache_read_input_token_cost": 1.71875e-07,
         "default_reasoning_effort": "none",

litellm/models/mcp_server.py

@@ -98,6 +98,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
     delegate_auth_to_upstream: bool = False
     oauth_passthrough: bool = False
     dcr_bridge: bool | None = None
+    per_server_oauth_discovery: bool = False
     is_byok: bool = False
     byok_description: list[str] = Field(default_factory=list)
     byok_api_key_help_url: str | None = None

litellm/ocr/main.py

@@ -191,6 +191,8 @@ def _prepare_ocr_request(
 def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool:
     if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native":
         return False
+    if not prepared_request.provider_config.supports_rust_bridge():
+        return False
     return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS
 
 

litellm/provider_endpoints_support_backup.json

@@ -559,6 +559,7 @@
         "moderations": false,
         "batches": false,
         "rerank": true,
+        "ocr": true,
         "a2a": true,
         "interactions": true
       }

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -197,7 +197,7 @@ def _gateway_dcr_challenge_target(
     if targets is None:
         return None
     server: Final = global_mcp_server_manager.get_mcp_server_by_name(targets[0], client_ip=client_ip)
-    if server is None or not server.is_gateway_managed_oauth2:
+    if server is None or not server.advertises_gateway_authorization_server:
         return None
     return targets[0]
 

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -1481,11 +1481,19 @@ async def _persist_dcr_client_registration(
         )
         updated_row: Final = await update_mcp_server(
             prisma_client=prisma_client,
-            data=UpdateMCPServerRequest(
-                server_id=mcp_server.server_id,
-                credentials=credentials,
-                oauth2_flow="authorization_code",
-                **({"token_url": mcp_server.token_url} if mcp_server.token_url else {}),
+            data=(
+                UpdateMCPServerRequest(
+                    server_id=mcp_server.server_id,
+                    credentials=credentials,
+                    oauth2_flow="authorization_code",
+                    token_url=mcp_server.token_url,
+                )
+                if mcp_server.token_url
+                else UpdateMCPServerRequest(
+                    server_id=mcp_server.server_id,
+                    credentials=credentials,
+                    oauth2_flow="authorization_code",
+                )
             ),
             touched_by="mcp_oauth_dcr",
         )
@@ -2367,7 +2375,7 @@ async def _build_oauth_protected_resource_response(
     if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange:
         _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource")
 
-    if explicitly_named and mcp_server is not None and mcp_server.is_gateway_managed_oauth2:
+    if explicitly_named and mcp_server is not None and mcp_server.advertises_gateway_authorization_server:
         return {
             "authorization_servers": [f"{request_base_url}/mcp"],
             "resource": resource_url,

litellm/proxy/_experimental/mcp_server/guardrail_translation/handler.py

@@ -1,16 +1,19 @@
 """
 MCP Guardrail Handler for Unified Guardrails.
 
-Converts an MCP call_tool (name + arguments) into a single OpenAI-compatible
-tool_call and passes it to apply_guardrail. Works with the synthetic payload
-from ProxyLogging._convert_mcp_to_llm_format.
+Converts an MCP call_tool (name + arguments) into the OpenAI-compatible shape
+apply_guardrail expects: the tool as a single-entry ``tools`` definition, and
+every string leaf of the call arguments as ``texts`` so text guardrails can
+detect and mask sensitive values in the payload. Works with the synthetic
+request from ProxyLogging._convert_mcp_to_llm_format.
 
 Note: For MCP tool definitions (schema) -> OpenAI tools=[], see
 litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_tool
 when you have a full MCP Tool from list_tools. Here we only have the call
-payload (name + arguments) so we just build the tool_call.
+payload (name + arguments) so we just build the tool definition.
 """
 
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 from fastapi import HTTPException
@@ -20,6 +23,8 @@ from litellm._logging import verbose_proxy_logger
 from litellm.experimental_mcp_client.tools import transform_mcp_tool_to_openai_tool
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
 from litellm.proxy._experimental.mcp_server.utils import (
+    MAX_STRUCTURED_CONTENT_SCAN_DEPTH,
+    JSONLeafPath,
     json_string_leaves,
     json_unrewritable_labels,
     mcp_content_item_text,
@@ -42,6 +47,72 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
 
+def _blocked(reason: str) -> HTTPException:
+    return HTTPException(status_code=400, detail={"error": f"Content blocked: {reason}"})
+
+
+def _too_deeply_nested() -> HTTPException:
+    return _blocked(
+        f"MCP tool call arguments exceed the maximum nesting depth of {MAX_STRUCTURED_CONTENT_SCAN_DEPTH} "
+        "and cannot be scanned by the configured guardrail"
+    )
+
+
+def _argument_replacements(
+    argument_leaves: tuple[tuple[JSONLeafPath, str], ...],
+    masked_texts: Sequence[str] | None,
+) -> Mapping[JSONLeafPath, str]:
+    """Positionally pair the guardrail's returned texts with the leaves they came from.
+
+    Only leaves the guardrail actually rewrote are returned, so a guardrail that
+    detects nothing leaves the outbound tool call byte-identical. A guardrail that
+    returns the wrong number of texts fails closed, because a positional write-back
+    would scramble the arguments rather than mask them.
+    """
+    if masked_texts is not None and len(masked_texts) != len(argument_leaves):
+        raise _blocked(
+            f"guardrail returned {len(masked_texts)} texts for {len(argument_leaves)} MCP tool call argument strings, "
+            "so the redaction cannot be mapped back to the arguments"
+        )
+    return {path: masked for (path, original), masked in zip(argument_leaves, masked_texts or ()) if masked != original}
+
+
+def _conflicting_rewrite_paths(
+    scanned_leaves: tuple[tuple[JSONLeafPath, str], ...],
+    current_leaves: tuple[tuple[JSONLeafPath, str], ...],
+    replacements: Mapping[JSONLeafPath, str],
+) -> tuple[JSONLeafPath, ...]:
+    """Paths another guardrail already rewrote differently from what this one wants.
+
+    Guardrails opted into ``run_in_parallel`` all scan the same payload snapshot, so
+    each one returns a full replacement string derived from the *original* leaf. Two
+    of them rewriting one leaf to different values cannot be merged: writing either
+    result discards the other guardrail's redaction. A leaf still holding the text
+    this guardrail was handed, or already holding this guardrail's own replacement,
+    is safe to write; the latter is how a guardrail that masks the arguments itself
+    as well as through ``texts`` gets there first. Anything else fails closed,
+    including a payload reshaped so the leaves no longer line up, because the
+    write-back is positional and would land a redaction on the wrong value.
+    """
+    if tuple(path for path, _ in scanned_leaves) != tuple(path for path, _ in current_leaves):
+        return tuple(replacements)
+    return tuple(
+        path
+        for (path, scanned), (_, current) in zip(scanned_leaves, current_leaves)
+        if path in replacements and current not in (scanned, replacements[path])
+    )
+
+
+def _conflicting_rewrite(paths: tuple[JSONLeafPath, ...]) -> HTTPException:
+    return _blocked(
+        "two guardrails running concurrently rewrote the same MCP tool call "
+        f"argument{'s' if len(paths) > 1 else ''} "
+        f"({', '.join('.'.join(str(part) for part in path) for path in paths)}); "
+        "their redactions cannot be merged. Remove run_in_parallel from one of them so they "
+        "run in sequence."
+    )
+
+
 class MCPGuardrailTranslationHandler(BaseTranslation):
     """Guardrail translation handler for MCP tool calls (passes a single tool_call to guardrail)."""
 
@@ -52,10 +123,8 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> dict[str, Any]:
         mcp_tool_name: Final = data.get("mcp_tool_name") or data.get("name")
-        mcp_arguments = data.get("mcp_arguments") or data.get("arguments")
+        mcp_arguments: Final[object] = data.get("mcp_arguments") or data.get("arguments")
         mcp_tool_description: Final = data.get("mcp_tool_description") or data.get("description")
-        if mcp_arguments is None or not isinstance(mcp_arguments, dict):
-            mcp_arguments = {}
 
         if not mcp_tool_name:
             verbose_proxy_logger.debug("MCP Guardrail: mcp_tool_name missing")
@@ -84,16 +153,37 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
                 strict=fn.get("strict", False) or False,  # Default to False if None
             ),
         }
+        argument_leaves: Final = json_string_leaves(mcp_arguments)
+        if argument_leaves is None:
+            raise _too_deeply_nested()
         inputs: Final[GenericGuardrailAPIInputs] = GenericGuardrailAPIInputs(
             tools=[tool_def],
+            texts=[text for _, text in argument_leaves],
         )
 
-        await guardrail_to_apply.apply_guardrail(
+        guarded: Final = await guardrail_to_apply.apply_guardrail(
             inputs=inputs,
             request_data=data,
             input_type="request",
             logging_obj=litellm_logging_obj,
         )
+        replacements: Final = _argument_replacements(
+            argument_leaves=argument_leaves,
+            masked_texts=guarded.get("texts") if guarded else None,
+        )
+        if not replacements:
+            return data
+
+        current_arguments: Final[object] = data.get("mcp_arguments") or data.get("arguments")
+        current_leaves: Final = json_string_leaves(current_arguments)
+        if current_leaves is None:
+            raise _too_deeply_nested()
+        conflicting: Final = _conflicting_rewrite_paths(argument_leaves, current_leaves, replacements)
+        if conflicting:
+            raise _conflicting_rewrite(conflicting)
+        masked_arguments: Final = with_json_string_leaves(current_arguments, replacements)
+        data["mcp_arguments"] = masked_arguments  # rebind-ok: preserve the mask for the outbound MCP call
+        data["modified_arguments"] = masked_arguments  # rebind-ok: expose the applied mask to the caller
         return data
 
     async def process_output_response(
@@ -131,14 +221,8 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
         structured_leaves: Final = json_string_leaves(structured) if structured is not None else ()
         structured_labels: Final = json_unrewritable_labels(structured) if structured is not None else ()
         if structured_leaves is None or structured_labels is None:
-            raise HTTPException(
-                status_code=400,
-                detail={
-                    "error": (
-                        "Content blocked: MCP tool result structuredContent is nested too deeply to be scanned "
-                        "by the configured guardrail"
-                    )
-                },
+            raise _blocked(
+                "MCP tool result structuredContent is nested too deeply to be scanned by the configured guardrail"
             )
 
         if not text_blocks and not structured_leaves and not structured_labels:
@@ -158,12 +242,10 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
         if masked_texts is None:
             return response
         if len(masked_texts) != len(originals):
-            verbose_proxy_logger.warning(
-                "MCP Guardrail: guardrail returned %d texts for %d tool result texts; leaving the result unmasked",
-                len(masked_texts),
-                len(originals),
+            raise _blocked(
+                f"guardrail returned {len(masked_texts)} texts for {len(originals)} MCP tool result texts, "
+                "so the redaction cannot be mapped back to the result"
             )
-            return response
 
         split: Final = len(text_blocks)
         if content is not None:
@@ -173,15 +255,10 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
 
         label_start: Final = split + len(structured_leaves)
         if any(masked != original for original, masked in zip(structured_labels, masked_texts[label_start:])):
-            raise HTTPException(
-                status_code=400,
-                detail={
-                    "error": (
-                        "Content blocked: MCP tool result matched a masking rule on a non-rewritable field "
-                        "(a structuredContent key or numeric value), which cannot be redacted without changing "
-                        "the payload contract"
-                    )
-                },
+            raise _blocked(
+                "MCP tool result matched a masking rule on a non-rewritable field "
+                "(a structuredContent key or numeric value), which cannot be redacted without changing "
+                "the payload contract"
             )
 
         structured_replacements: Final = {

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -155,10 +155,14 @@ from litellm.proxy._types import (
     MCPTransportType,
     SpecialMCPServerNames,
     UserAPIKeyAuth,
+    is_per_server_oauth_discovery_eligible,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
 from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
+from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import (
+    id_jag_assertion_capture_gap_at_startup,
+)
 from litellm.proxy.utils import PrismaClient, ProxyLogging, get_server_root_path
 from litellm.repositories.table_repositories import MCPServerRepository
 from litellm.types.llms.custom_http import httpxSpecialProvider
@@ -341,6 +345,7 @@ class MCPServerConfig(TypedDict, total=False):
     token_endpoint_auth_method: MCPTokenEndpointAuthMethod
     scopes: str | Sequence[str]
     dcr_bridge: object
+    per_server_oauth_discovery: ReadOnly[object]
     extra_headers: _StringList
     allowed_tools: _StringList
     disallowed_tools: _StringList
@@ -411,6 +416,31 @@ def _blank_to_none(value: str | None) -> str | None:
     return value.strip() or None
 
 
+def _config_per_server_oauth_discovery(
+    server_config: MCPServerConfig,
+    server_ref: str,
+    auth_type: MCPAuthType | None,
+    oauth2_flow: object,
+) -> bool:
+    match server_config.get("per_server_oauth_discovery", False):
+        case bool() as enabled:
+            pass
+        case other:
+            raise ValueError(
+                f"Invalid config for MCP server '{server_ref}': per_server_oauth_discovery must be a boolean "
+                f"(got {other!r})."
+            )
+    relay_eligible: Final = is_per_server_oauth_discovery_eligible(
+        auth_type, oauth2_flow, server_config.get("delegate_auth_to_upstream", False)
+    )
+    if enabled and not relay_eligible:
+        raise ValueError(
+            f"Invalid config for MCP server '{server_ref}': per_server_oauth_discovery is only supported for "
+            "auth_type oauth2 with oauth2_flow authorization_code and without delegate_auth_to_upstream."
+        )
+    return enabled
+
+
 def _pinned_config_server_id(raw_server_id: object, server_name: str) -> str | None:
     """Return the ``server_id`` an admin pinned for this config.yaml server, or ``None`` when absent.
 
@@ -1382,6 +1412,20 @@ def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str
     )
 
 
+def _warn_config_id_jag_server_outruns_sso(server: MCPServer) -> None:
+    if server.auth_type != MCPAuth.oauth2_id_jag:
+        return
+    gap: Final = id_jag_assertion_capture_gap_at_startup()
+    if gap is None:
+        return
+    verbose_logger.warning(
+        "MCP server %r (id=%s, source=config) is declared with auth_type=oauth2_id_jag, but %s.",
+        get_server_prefix(server),
+        server.server_id,
+        gap,
+    )
+
+
 def _deserialize_json_dict(data: str | _StringMap | None) -> dict[str, str] | None:
     """
     Deserialize optional JSON mappings stored in the database.
@@ -2290,6 +2334,9 @@ class MCPServerManager:
                 )
 
             config_dcr_bridge = server_config.get("dcr_bridge", None)
+            config_per_server_oauth_discovery = _config_per_server_oauth_discovery(
+                server_config, server_name or server_id, auth_type, config_oauth2_flow
+            )
             if config_dcr_bridge is not None and not isinstance(config_dcr_bridge, bool):
                 raise ValueError(
                     f"Invalid config for MCP server '{server_name or server_id}': dcr_bridge "
@@ -2361,6 +2408,7 @@ class MCPServerManager:
                 delegate_auth_to_upstream=bool(server_config.get("delegate_auth_to_upstream", False)),
                 oauth_passthrough=bool(server_config.get("oauth_passthrough", False)),
                 dcr_bridge=config_dcr_bridge,
+                per_server_oauth_discovery=config_per_server_oauth_discovery,
                 # AWS SigV4 fields
                 aws_access_key_id=server_config.get("aws_access_key_id", None),
                 aws_secret_access_key=server_config.get("aws_secret_access_key", None),
@@ -2393,6 +2441,7 @@ class MCPServerManager:
             )
             self._assign_unique_short_prefix(new_server)
             _warn_internal_delegate_pkce_if_applicable(new_server, source="config")
+            _warn_config_id_jag_server_outruns_sso(new_server)
             self.config_mcp_servers[server_id] = new_server
             self._set_oauth_discovery_deferred(
                 server_id,
@@ -2885,6 +2934,7 @@ class MCPServerManager:
             delegate_auth_to_upstream=bool(getattr(mcp_server, "delegate_auth_to_upstream", False)),
             oauth_passthrough=bool(getattr(mcp_server, "oauth_passthrough", False)),
             dcr_bridge=getattr(mcp_server, "dcr_bridge", None),
+            per_server_oauth_discovery=bool(getattr(mcp_server, "per_server_oauth_discovery", False)),
             created_at=getattr(mcp_server, "created_at", None),
             updated_at=getattr(mcp_server, "updated_at", None),
             tool_name_to_display_name=_deserialize_json_dict(getattr(mcp_server, "tool_name_to_display_name", None)),
@@ -6674,6 +6724,7 @@ class MCPServerManager:
             registration_url=server.configured_registration_url or server.registration_url,
             oauth2_flow=server.oauth2_flow,
             dcr_bridge=server.dcr_bridge,
+            per_server_oauth_discovery=server.per_server_oauth_discovery,
             token_exchange_endpoint=server.token_exchange_endpoint,
             audience=server.audience,
             subject_token_type=server.subject_token_type,
@@ -6792,6 +6843,7 @@ class MCPServerManager:
             delegate_auth_to_upstream=server.delegate_auth_to_upstream,
             oauth_passthrough=getattr(server, "oauth_passthrough", False),
             dcr_bridge=server.dcr_bridge,
+            per_server_oauth_discovery=server.per_server_oauth_discovery,
             is_byok=server.is_byok,
             byok_description=server.byok_description,
             byok_api_key_help_url=server.byok_api_key_help_url,

litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py

@@ -31,11 +31,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_sto
     TokenCacheBackend,
     TokenStoreUnavailable,
 )
-from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_distributed_lock import (
-    RedisDistributedLock,
-)
-from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import (
-    RedisRefreshCoordinator,
+from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import (
+    runtime_refresh_coordinator,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import (
     OAuthTokenCacheCodec,
@@ -131,23 +128,17 @@ def _runtime_backend_and_coordinator() -> tuple[TokenCacheBackend | None, Refres
     )
     from litellm.proxy.proxy_server import user_api_key_cache  # noqa: PLC0415
 
-    redis_cache: Final = user_api_key_cache.redis_cache
-    if redis_cache is None:
+    coordinator: Final = runtime_refresh_coordinator()
+    if coordinator is None:
         return None, None, False
     codec: Final = OAuthTokenCacheCodec(
         encrypt_value_helper,
         lambda blob: decrypt_value_helper(blob, "mcp_per_user_token", exception_type="debug"),
     )
-    # user_api_key_cache satisfies the AsyncCache slice (DualCache types ttl via **kwargs) and the
-    # Redis client from init_async_client() is partially typed - both are untyped-boundary casts.
+    # user_api_key_cache satisfies the AsyncCache slice (DualCache types ttl via **kwargs) - an
+    # untyped-boundary cast.
     cache: Final[AsyncCache] = user_api_key_cache  # pyright: ignore
-    redis_client: Final = redis_cache.init_async_client()  # pyright: ignore
-    lock: Final = RedisDistributedLock(
-        redis_client,  # pyright: ignore
-        namespace_key=redis_cache.check_and_fix_namespace,
-    )
     backend: Final = DualCacheTokenCacheBackend(cache, codec)
-    coordinator: Final = RedisRefreshCoordinator(lock)
     return backend, coordinator, True
 
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py

@@ -46,11 +46,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
     Ok,
     Result,
 )
+from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import (
+    default_sso_assertion_store,
+)
 from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import (
     AssertionStoreUnavailable,
-    DbSSOAssertionStore,
     SSOAssertionStore,
-    SSOIdentityAssertion,
+    assertion_expired,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import (
     ExchangedToken,
@@ -129,7 +131,7 @@ class UpstreamCredentialProvider:
         self._token_endpoint: TokenEndpointClient = token_endpoint or TokenEndpointClient()
         self._exchanged_tokens: ExchangedTokenCache = exchanged_tokens or ExchangedTokenCache()
         self._client_credentials_source = client_credentials_source or ClientCredentialsTokenSource()
-        self._sso_assertion_store: SSOAssertionStore = sso_assertion_store or DbSSOAssertionStore()
+        self._sso_assertion_store: SSOAssertionStore = sso_assertion_store or default_sso_assertion_store()
 
     async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]:
         match server.config:
@@ -246,7 +248,7 @@ class UpstreamCredentialProvider:
                     "Sign in through LiteLLM SSO so the gateway captures one."
                 )
             )
-        if _assertion_expired(assertion, datetime.now(timezone.utc)):
+        if assertion_expired(assertion, datetime.now(timezone.utc)):
             return Error(
                 CredError.of_precondition_required(
                     "The stored IdP identity assertion for this user has expired. Sign in through "
@@ -405,19 +407,6 @@ def _id_jag_slot_key(subject: Subject, server: ServerSpec) -> str:
     return hashlib.sha256(material.encode()).hexdigest()
 
 
-def _assertion_expired(assertion: SSOIdentityAssertion, now: datetime) -> bool:
-    """Whether the stored assertion's ``exp`` has passed. An assertion carrying no expiry is
-    treated as usable and left for the IdP to reject, since the store records what the id_token
-    claimed rather than imposing a lifetime of its own. A naive ``expires_at`` is read as UTC so a
-    stored value that lost its offset compares instead of raising.
-    """
-    expires_at: Final = assertion.expires_at
-    if expires_at is None:
-        return False
-    normalized: Final = expires_at if expires_at.tzinfo is not None else expires_at.replace(tzinfo=timezone.utc)
-    return normalized <= now
-
-
 def _id_jag_fingerprint(subject_token: str, server_id: str, config: IdJagConfig) -> str:
     """What the cached leg-2 bearer was minted from: the subject token, the server, and the config.
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/runtime_refresh_coordinator.py

@@ -0,0 +1,41 @@
+"""The runtime ``RefreshCoordinator``: cross-replica single-flight when Redis is wired.
+
+Builds ``RedisRefreshCoordinator`` over the proxy's shared Redis so one refresh runs per key
+across the fleet, or returns ``None`` when Redis is absent so the caller keeps the foundation's
+in-process default (correct for a single replica). The proxy globals it reads are not ready at
+import time, so this is called per composition rather than held as module state.
+
+Shared by every credential arm that renews a stored grant: a rotating refresh token must be
+redeemed once across all workers, so each arm electing its own winner with its own lock shape
+would be a bug waiting to differ.
+"""
+
+from __future__ import annotations
+
+from typing import Final
+
+from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
+    RefreshCoordinator,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_distributed_lock import (
+    RedisDistributedLock,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import (
+    RedisRefreshCoordinator,
+)
+
+
+def runtime_refresh_coordinator() -> RefreshCoordinator | None:
+    from litellm.proxy.proxy_server import user_api_key_cache  # noqa: PLC0415  # runtime global
+
+    redis_cache: Final = user_api_key_cache.redis_cache
+    if redis_cache is None:
+        return None
+    # The Redis client from init_async_client() is only partially typed; the lock validates every
+    # reply it depends on, so the untyped boundary is contained here.
+    redis_client: Final = redis_cache.init_async_client()  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]  # litellm redis wrapper is untyped
+    lock: Final = RedisDistributedLock(
+        redis_client,  # pyright: ignore[reportArgumentType,reportUnknownArgumentType]  # litellm redis wrapper is untyped
+        namespace_key=redis_cache.check_and_fix_namespace,
+    )
+    return RedisRefreshCoordinator(lock)

litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_refresher.py

@@ -0,0 +1,469 @@
+"""Renew the stored SSO identity assertion so an ID-JAG agent outlives one id_token.
+
+The ``oauth2_id_jag`` arm asserts the id_token captured at the user's last interactive sign-in, so
+without renewal an agent holding a brokered LiteLLM key can act for that user only until that token's
+``exp``, typically an hour, and the sole recovery is another interactive login. The assertion already
+carries the IdP refresh token beside it; this module is what redeems it.
+
+``RefreshingSSOAssertionStore`` wraps any ``SSOAssertionStore`` and satisfies the same protocol, so
+the egress arm is unchanged: it still reads one assertion and still judges expiry itself. Renewal is
+lazy (only a read that finds a near-expiry assertion triggers one, so IdP traffic tracks actual use,
+not the size of the user table) and single-flighted per user through the same ``RefreshCoordinator``
+the ``authorization_code`` arm uses, because an IdP that rotates refresh tokens treats two concurrent
+redemptions of one token as replay and can revoke the whole grant chain.
+
+The refresh is redeemed against the generic-OIDC client the login itself used
+(``GENERIC_TOKEN_ENDPOINT`` / ``GENERIC_CLIENT_ID`` / ``GENERIC_CLIENT_SECRET``, which the proxy
+reconciles from the stored SSO row into the process environment at startup), authenticated the way
+that login authenticated: the non-PKCE path always sends HTTP Basic, while the PKCE path sends the
+credentials in the body when ``GENERIC_INCLUDE_CLIENT_ID`` is set, and an IdP application may accept
+only one of the two. An assertion can only exist if that client minted it, so no other client could
+redeem its refresh token, and no other method is known to be accepted. A deployment whose
+``GENERIC_SCOPE`` omits ``offline_access`` captures no refresh token at all, which is why that miss
+logs the scope by name rather than failing silently.
+
+Failures are values internally (``Result[_, RefreshFailure]``). At the store boundary they collapse
+onto the protocol's existing two-outcome contract: a refusal returns the expired assertion unchanged
+so the reader's own guard challenges the user to sign in again, while a transient IdP failure raises
+``AssertionStoreUnavailable`` so the reader answers 503 instead of blaming the user for an outage.
+
+One ambiguity remains under Redis-coordinated renewal across replicas. A cross-replica loser that
+finds the row still expiring after the holder finished cannot tell a refused refresh from a renewal
+that could not be recorded. Redeeming itself could consume a refresh token the holder may already
+have rotated, so it answers retryable 503 rather than guessing a sign-in challenge. The next
+uncontended read settles the outcome itself: a refusal challenges, and a successful refresh persists.
+If the holder rotated the token but its write failed, that rotation is lost and the next uncontended
+read's refusal challenges, which is the only honest answer because the rotated token was never
+recorded. On the refusal path, the loser pays for one retry before that challenge.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Final, Literal, Protocol
+
+import httpx
+from pydantic import SecretStr, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm._logging import verbose_proxy_logger
+from litellm.exceptions import Timeout
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # litellm http handler is untyped
+)
+from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import (
+    build_token_endpoint_client_auth,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
+    InProcessRefreshCoordinator,
+    RefreshCoordinator,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+    Error,
+    Ok,
+    Result,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import (
+    runtime_refresh_coordinator,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import (
+    AssertionStoreUnavailable,
+    DbSSOAssertionStore,
+    SSOAssertionStore,
+    SSOIdentityAssertion,
+    assertion_expired,
+    assertion_from_sso_login,
+    fetch_sso_identity_assertion,
+    persist_sso_identity_assertion,
+)
+from litellm.types.llms.custom_http import httpxSpecialProvider
+from litellm.types.mcp import MCPTokenEndpointAuthMethod
+
+_BODY_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(dict[str, object])
+
+_REFRESH_GRANT_TYPE: Final = "refresh_token"
+# The lock namespace for the one assertion row a user has; the sibling arm keys the same lock by
+# server_id, and no server_id can collide with this literal.
+_SINGLE_FLIGHT_KEY: Final = "sso_identity_assertion"
+# Renew this far ahead of ``exp`` so a token that would die between resolution and the second leg of
+# the exchange is replaced first. Matches the sibling per-user token store's skew.
+_DEFAULT_EXPIRY_SKEW_SECONDS: Final = 60.0
+
+
+class AssertionRead(Protocol):
+    """Reads the user's stored assertion row."""
+
+    async def __call__(self, user_id: str) -> SSOIdentityAssertion | None: ...
+
+
+class AssertionWrite(Protocol):
+    """Replaces the user's stored assertion row."""
+
+    async def __call__(self, user_id: str, assertion: SSOIdentityAssertion) -> None: ...
+
+
+class CoordinatorFactory(Protocol):
+    """Builds the cross-replica coordinator, or ``None`` when there is no shared lock to build on."""
+
+    def __call__(self) -> RefreshCoordinator | None: ...
+
+
+class FormPost(Protocol):
+    """POSTs an OAuth form and hands back the raw response."""
+
+    async def __call__(
+        self, url: str, form: Mapping[str, str], headers: Mapping[str, str]
+    ) -> httpx.Response | None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class SSOClientConfig:
+    """The generic-OIDC client credentials a refresh_token grant has to authenticate as, and how."""
+
+    token_endpoint: str
+    client_id: str
+    client_secret: SecretStr
+    auth_method: MCPTokenEndpointAuthMethod
+
+
+def sso_client_config(env: Mapping[str, str]) -> SSOClientConfig | None:
+    """The configured generic-OIDC client, or ``None`` when the deployment has none.
+
+    Read from the process environment because that is where the login path reads it
+    (``_setup_generic_sso_env_vars``) and where the proxy materializes the stored ``sso_config`` row
+    at startup, so this resolves to the same client that minted the assertion. ``None`` is an
+    ordinary state, not an error: a deployment signing in through a provider that captures no
+    assertion has nothing here to renew, and a client with no secret is not a confidential client
+    that could redeem one.
+
+    ``auth_method`` is derived from the same ``GENERIC_INCLUDE_CLIENT_ID`` the login reads, because
+    the two login paths do not agree: the non-PKCE path always authenticates with HTTP Basic, while
+    the PKCE path puts the credentials in the body when that flag is set. Both capture assertions, so
+    a constant here would authenticate the renewal differently from the sign-in that produced the
+    refresh token and 401 against an IdP application registered for only one of the two.
+    """
+    token_endpoint: Final = env.get("GENERIC_TOKEN_ENDPOINT")
+    client_id: Final = env.get("GENERIC_CLIENT_ID")
+    client_secret: Final = env.get("GENERIC_CLIENT_SECRET")
+    if not token_endpoint or not client_id or not client_secret:
+        return None
+    includes_client_id: Final = env.get("GENERIC_INCLUDE_CLIENT_ID", "false").lower() == "true"
+    return SSOClientConfig(
+        token_endpoint=token_endpoint,
+        client_id=client_id,
+        client_secret=SecretStr(client_secret),
+        auth_method="client_secret_post" if includes_client_id else "client_secret_basic",
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshFailure:
+    """Why a renewal produced nothing, split by what the caller can do about it.
+
+    ``rejected`` is settled: this refresh token will never work again, so the user has to sign in.
+    ``unavailable`` is transient: the same attempt may succeed in a minute, so telling the user to
+    sign in again would be a lie about whose problem it is. Both arms carry the same payload, so
+    this is a ``Literal`` discriminant rather than a ``tagged_union``; consumers still ``match`` on
+    ``kind`` with an ``assert_never`` tail.
+    """
+
+    kind: Literal["rejected", "unavailable"]
+    detail: str
+
+    @staticmethod
+    def of_rejected(detail: str) -> RefreshFailure:
+        return RefreshFailure(kind="rejected", detail=detail)
+
+    @staticmethod
+    def of_unavailable(detail: str) -> RefreshFailure:
+        return RefreshFailure(kind="unavailable", detail=detail)
+
+
+class TokenEndpointTransport(Protocol):
+    """One form POST to the IdP token endpoint, with the refusal/outage split preserved.
+
+    That split is the whole reason this is not the resolver's ``TokenEndpointClient``: that
+    collaborator maps every non-2xx to ``upstream_unavailable``, which is right for an exchange leg
+    and wrong here, where a 400 ``invalid_grant`` means the stored refresh token is dead and the user
+    must act.
+    """
+
+    async def post(
+        self, url: str, form: Mapping[str, str], headers: Mapping[str, str]
+    ) -> Result[Mapping[str, object], RefreshFailure]: ...
+
+
+async def post_form(url: str, form: Mapping[str, str], headers: Mapping[str, str]) -> httpx.Response | None:
+    # litellm's httpx handler is only partially typed; nothing but the response object crosses back,
+    # and the transport below validates its body, so the untyped boundary is contained here.
+    client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)  # pyright: ignore[reportUnknownVariableType]  # litellm http handler is untyped
+    return await client.post(url, data=form, headers=headers)  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType,reportReturnType,reportArgumentType]  # litellm http handler is untyped and its stub narrows data=/headers= to dict, which httpx itself does not require
+
+
+class HttpxTokenEndpointTransport:
+    """The live transport. 4xx is the IdP refusing this grant; anything else is an outage.
+
+    The POST itself is injected so that split, which decides whether the user is challenged or told
+    to wait, is testable without a live IdP.
+    """
+
+    def __init__(self, post: FormPost = post_form) -> None:
+        self._post = post
+
+    async def post(
+        self, url: str, form: Mapping[str, str], headers: Mapping[str, str]
+    ) -> Result[Mapping[str, object], RefreshFailure]:
+        try:
+            response: Final = await self._post(url, form, headers)
+            if response is None:
+                return Error(RefreshFailure.of_unavailable("the IdP token endpoint returned no response"))
+            response.raise_for_status()
+            body: Final = _BODY_ADAPTER.validate_python(response.json())  # pyright: ignore[reportAny]  # untyped JSON; the adapter is the type gate
+        except httpx.HTTPStatusError as exc:
+            status: Final = exc.response.status_code
+            if 400 <= status < 500:
+                return Error(RefreshFailure.of_rejected(f"the IdP refused the refresh with status {status}"))
+            return Error(RefreshFailure.of_unavailable(f"the IdP token endpoint answered with status {status}"))
+        except (httpx.RequestError, Timeout) as exc:
+            return Error(RefreshFailure.of_unavailable(f"the IdP token endpoint is unreachable ({type(exc).__name__})"))
+        except json.JSONDecodeError:
+            return Error(RefreshFailure.of_unavailable("the IdP token endpoint returned a non-JSON response"))
+        except ValidationError:
+            return Error(RefreshFailure.of_unavailable("the IdP token endpoint returned a non-object response"))
+        return Ok(body)
+
+
+class SSOAssertionRefresher:
+    """Redeems the stored refresh token for a current id_token and writes the rotation back.
+
+    Collaborators are injected so the orchestration, the untyped response parsing and the
+    write-back race are all testable without an IdP or a database.
+    """
+
+    def __init__(
+        self,
+        transport: TokenEndpointTransport,
+        *,
+        client_config: Callable[[], SSOClientConfig | None] = lambda: sso_client_config(os.environ),
+        read: AssertionRead = fetch_sso_identity_assertion,
+        write: AssertionWrite = persist_sso_identity_assertion,
+    ) -> None:
+        self._transport = transport
+        self._client_config = client_config
+        self._read = read
+        self._write = write
+
+    async def refresh(
+        self, user_id: str, assertion: SSOIdentityAssertion
+    ) -> Result[SSOIdentityAssertion, RefreshFailure]:
+        if assertion.refresh_token is None:
+            verbose_proxy_logger.warning(
+                "ID-JAG: the stored IdP identity assertion for user_id=%s has expired and no refresh token was "
+                "captured with it, so it cannot be renewed without another interactive sign-in. Add "
+                "'offline_access' to GENERIC_SCOPE so the SSO login captures one.",
+                user_id,
+            )
+            return Error(RefreshFailure.of_rejected("no refresh token was captured at sign-in"))
+        config: Final = self._client_config()
+        if config is None:
+            verbose_proxy_logger.warning(
+                "ID-JAG: the stored IdP identity assertion for user_id=%s has expired and cannot be renewed "
+                "because the generic SSO client is not configured (GENERIC_TOKEN_ENDPOINT, GENERIC_CLIENT_ID, "
+                "GENERIC_CLIENT_SECRET).",
+                user_id,
+            )
+            return Error(RefreshFailure.of_rejected("the generic SSO client is not configured"))
+
+        carried_refresh_token: Final = assertion.refresh_token.get_secret_value()
+        # Whichever method the SSO login used for this client, since that is the one the IdP
+        # application is known to accept: an assertion only exists to renew because a sign-in already
+        # authenticated this client that way.
+        client_auth: Final = build_token_endpoint_client_auth(
+            auth_method=config.auth_method,
+            client_id=config.client_id,
+            client_secret=config.client_secret.get_secret_value(),
+        )
+        form: Final = {  # mutable-ok: the RFC 6749 form body is a wire format the HTTP client takes as a mapping
+            "grant_type": _REFRESH_GRANT_TYPE,
+            "refresh_token": carried_refresh_token,
+            **client_auth.body,
+        }
+        match await self._transport.post(config.token_endpoint, form, client_auth.headers):
+            case Error(failure):
+                return Error(failure)
+            case Ok(body):
+                return await self._renewed_from(user_id, assertion, body, carried_refresh_token)
+
+    async def _renewed_from(
+        self,
+        user_id: str,
+        previous: SSOIdentityAssertion,
+        body: Mapping[str, object],
+        carried_refresh_token: str,
+    ) -> Result[SSOIdentityAssertion, RefreshFailure]:
+        """The renewed assertion, built by the same validator the login path uses.
+
+        A rotated refresh token replaces the stored one; an omitted one carries forward, since an
+        IdP that does not rotate expects the original to keep working.
+        """
+        rotated: Final = body.get("refresh_token")
+        renewed: Final = assertion_from_sso_login(
+            body.get("id_token"),
+            rotated if isinstance(rotated, str) and rotated else carried_refresh_token,
+        )
+        if renewed is None:
+            verbose_proxy_logger.warning(
+                "ID-JAG: the IdP accepted the refresh for user_id=%s but returned no usable id_token, so there "
+                "is nothing to assert upstream. The SSO client's grant needs the 'openid' scope for the token "
+                "endpoint to return one on a refresh.",
+                user_id,
+            )
+            return Error(RefreshFailure.of_rejected("the IdP's refresh response carried no usable id_token"))
+        failure: Final = await self._store_renewal(user_id, previous, renewed)
+        if failure is not None:
+            return Error(failure)
+        return Ok(renewed)
+
+    async def _store_renewal(
+        self, user_id: str, previous: SSOIdentityAssertion, renewed: SSOIdentityAssertion
+    ) -> RefreshFailure | None:
+        """Write the renewal back, unless the row moved on while this renewal was in flight.
+
+        The row is one per user and last-write-wins, so an interactive sign-in landing mid-renewal
+        would otherwise be overwritten with a refresh token the IdP has already rotated away, costing
+        that user a sign-in later. Comparing against the id_token this renewal started from is what
+        detects that; skipping is safe because the newer row is the one the reader wants anyway.
+
+        A failed write is transient, not settled. The store, not this return value, is what every
+        caller reads, so a renewal that could not be recorded is a renewal nobody will see; saying so
+        keeps a database problem answering 503 rather than telling the user to sign in again over it.
+        """
+        try:
+            current: Final = await self._read(user_id)
+            if current is not None and current.id_token.get_secret_value() != previous.id_token.get_secret_value():
+                verbose_proxy_logger.info(
+                    "ID-JAG: a newer IdP identity assertion for user_id=%s was stored while this renewal was in "
+                    "flight; keeping the stored one.",
+                    user_id,
+                )
+                return None
+            await self._write(user_id, renewed)
+        except Exception as exc:  # noqa: BLE001  # any storage failure is transient here, never the user's fault
+            verbose_proxy_logger.warning(
+                "ID-JAG: could not persist the renewed IdP identity assertion for user_id=%s, so the rotated "
+                "refresh token is lost and this user will have to sign in again once the renewed token expires: %s",
+                user_id,
+                exc,
+            )
+            return RefreshFailure.of_unavailable("the renewed IdP identity assertion could not be persisted")
+        return None
+
+
+class RefreshingSSOAssertionStore:
+    """An ``SSOAssertionStore`` that renews a near-expiry assertion before handing it back.
+
+    Reads the inner store; an assertion still comfortably inside its lifetime is returned untouched,
+    so the common path costs exactly what it did before. Otherwise one renewal runs per user through
+    the injected ``RefreshCoordinator`` and every caller then re-reads the inner store, which is the
+    authority: the winner's write is what they all observe, and a renewal the write-back guard
+    skipped yields the newer assertion that displaced it rather than a private copy.
+
+    A refusal leaves the expired assertion in place for the reader's own guard to reject, so the user
+    sees the same sign-in-again challenge as before this store existed. A transient IdP failure
+    raises ``AssertionStoreUnavailable``, the protocol's existing signal for "this is not the user's
+    fault"; concurrent in-process callers share that outcome, while a cross-replica loser answers 503
+    when its re-read still finds the row expiring. On the refusal path that costs the loser one retry,
+    which then challenges. If the holder rotated the token but its write failed, the rotation is lost
+    and the next uncontended read's refusal challenges, the only honest answer because that token was
+    never recorded.
+    """
+
+    def __init__(
+        self,
+        inner: SSOAssertionStore,
+        refresher: SSOAssertionRefresher,
+        *,
+        fresh_read: AssertionRead,
+        coordinator_factory: CoordinatorFactory = runtime_refresh_coordinator,
+        expiry_skew_seconds: float = _DEFAULT_EXPIRY_SKEW_SECONDS,
+        clock: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
+    ) -> None:
+        self._inner = inner
+        self._refresher = refresher
+        self._fresh_read = fresh_read
+        self._coordinator_factory = coordinator_factory
+        self._in_process_coordinator = InProcessRefreshCoordinator()
+        self._distributed_coordinator: RefreshCoordinator | None = None
+        self._skew = timedelta(seconds=expiry_skew_seconds)
+        self._clock = clock
+
+    async def fetch(self, user_id: str) -> SSOIdentityAssertion | None:
+        assertion: Final = await self._inner.fetch(user_id)
+        if not self._expiring(assertion):
+            return assertion
+        await self._coordinator().run(
+            user_id,
+            _SINGLE_FLIGHT_KEY,
+            refresh=lambda: self._renew(user_id),
+            reread=lambda: self._reread_renewed(user_id),
+        )
+        return await self._fresh_read(user_id)
+
+    def _expiring(self, assertion: SSOIdentityAssertion | None) -> bool:
+        return assertion is not None and assertion_expired(assertion, self._clock() + self._skew)
+
+    def _coordinator(self) -> RefreshCoordinator:
+        """The cross-replica coordinator once Redis is reachable, else the in-process one.
+
+        Built on first use and kept, because the proxy's Redis client is not wired at import time;
+        retried while it is absent so a proxy that gains Redis later stops electing per-worker.
+        """
+        if self._distributed_coordinator is None:
+            self._distributed_coordinator = self._coordinator_factory()
+        return self._distributed_coordinator or self._in_process_coordinator
+
+    async def _renew(self, user_id: str) -> None:
+        """The elected renewal, judged from a fresh read so a rotation another replica just landed is
+        never redeemed again. Returns nothing: the inner store, not this return value, is what every
+        caller reads afterwards, so the winner and the losers cannot disagree."""
+        latest: Final = await self._fresh_read(user_id)
+        if latest is None or not self._expiring(latest):
+            return
+        match await self._refresher.refresh(user_id, latest):
+            case Ok(_):
+                return
+            case Error(failure):
+                match failure.kind:
+                    case "rejected":
+                        return
+                    case "unavailable":
+                        raise AssertionStoreUnavailable(failure.detail)
+                assert_never(failure.kind)
+
+    async def _reread_renewed(self, user_id: str) -> None:
+        """A loser cannot distinguish refusal from an unrecorded renewal without risking token replay.
+
+        It answers retryable 503 instead of guessing a sign-in challenge; the retry runs uncontended
+        and settles the outcome itself.
+        """
+        latest: Final = await self._fresh_read(user_id)
+        if self._expiring(latest):
+            raise AssertionStoreUnavailable(
+                f"the IdP identity assertion for user_id={user_id} was being renewed by another replica "
+                "and is not yet current; retry shortly"
+            )
+
+
+def default_sso_assertion_store() -> SSOAssertionStore:
+    """The live read seam for the ``id_jag`` arm: the stored assertion, renewed when it is stale."""
+    db_store: Final = DbSSOAssertionStore()
+    fresh_read: Final = db_store.fetch_uncached
+    return RefreshingSSOAssertionStore(
+        db_store,
+        SSOAssertionRefresher(HttpxTokenEndpointTransport(), read=fresh_read),
+        fresh_read=fresh_read,
+    )

litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py

@@ -127,6 +127,23 @@ def assertion_from_sso_login(id_token: object, refresh_token: object) -> SSOIden
     )
 
 
+def assertion_expired(assertion: SSOIdentityAssertion, now: datetime) -> bool:
+    """Whether the assertion's ``exp`` has passed at ``now``. An assertion carrying no expiry is
+    treated as usable and left for the IdP to reject, since the store records what the id_token
+    claimed rather than imposing a lifetime of its own. A naive ``expires_at`` is read as UTC so a
+    stored value that lost its offset compares instead of raising.
+
+    Lives beside the model rather than in either reader so the egress guard and the renewal
+    trigger judge the same field the same way; passing a ``now`` in the future is how a caller
+    asks "is this about to expire" without a second, driftable predicate.
+    """
+    expires_at: Final = assertion.expires_at
+    if expires_at is None:
+        return False
+    normalized: Final = expires_at if expires_at.tzinfo is not None else expires_at.replace(tzinfo=timezone.utc)
+    return normalized <= now
+
+
 async def ema_assertion_retention_enabled() -> bool:
     """Whether any MCP server uses ``oauth2_id_jag``, evaluated per login so the gateway only
     retains bearer material while an EMA upstream exists to spend it on. Judged against the two
@@ -146,7 +163,9 @@ async def ema_assertion_retention_enabled() -> bool:
         return True
     if prisma_client is None:
         return False
-    row = await prisma_client.db.litellm_mcpservertable.find_first(where={"auth_type": MCPAuth.oauth2_id_jag.value})
+    row: Final = await prisma_client.db.litellm_mcpservertable.find_first(
+        where={"auth_type": MCPAuth.oauth2_id_jag.value}
+    )
     return row is not None
 
 
@@ -158,7 +177,7 @@ async def persist_sso_identity_assertion(
 
     if prisma_client is None:
         return
-    payload: Final[dict[str, str]] = {
+    payload: Final = {
         "id_token": assertion.id_token.get_secret_value(),
         **({"refresh_token": assertion.refresh_token.get_secret_value()} if assertion.refresh_token else {}),
         **({"issuer": assertion.issuer} if assertion.issuer else {}),
@@ -220,11 +239,13 @@ async def fetch_sso_identity_assertion(
 
 
 class AssertionStoreUnavailable(Exception):
-    """Raised by ``fetch`` when the backing store is unreachable (e.g. the DB is down).
+    """Raised by ``fetch`` when the assertion cannot be read for a transient reason: the DB is
+    down, or the IdP behind a renewing store could not be reached.
 
     Distinct from returning ``None`` for "this user has no captured assertion": an outage must not
     read as a definite absence, which would tell the user to sign in again over a transient failure,
-    and it must not escape as an unhandled error on the egress or retry path. Mirrors
+    and it must not escape as an unhandled error on the egress or retry path. The message names the
+    real component for the operator log; callers get the reader's generic 503. Mirrors
     ``TokenStoreUnavailable`` on the sibling per-user OAuth store.
     """
 
@@ -257,6 +278,12 @@ class DbSSOAssertionStore:
         except Exception as exc:  # noqa: BLE001  # any driver/storage failure is an outage, not an absence
             raise AssertionStoreUnavailable(str(exc)) from exc
 
+    async def fetch_uncached(self, user_id: str) -> SSOIdentityAssertion | None:
+        try:
+            return await _read_assertion_from_db(user_id)
+        except Exception as exc:  # noqa: BLE001  # any driver/storage failure is an outage, not an absence
+            raise AssertionStoreUnavailable(str(exc)) from exc
+
 
 async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient, new_master_key: str) -> None:
     """Re-encrypt every stored assertion under ``new_master_key`` during a salt-key rotation,
@@ -280,7 +307,9 @@ async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient,
                 row.user_id,
             )
             return False
-        re_encrypted = _STR_ADAPTER.validate_python(encrypt_value_helper(plaintext, new_encryption_key=new_master_key))
+        re_encrypted: Final = _STR_ADAPTER.validate_python(
+            encrypt_value_helper(plaintext, new_encryption_key=new_master_key)
+        )
         await prisma_client.db.litellm_ssoidentityassertion.update(
             where={"user_id": row.user_id},
             data={"assertion_b64": re_encrypted},

litellm/proxy/_experimental/mcp_server/utils.py

@@ -752,7 +752,7 @@ def interpolate_headers(headers: Mapping[str, str], variables: Mapping[str, str]
 def build_env_var_setup_url(server_id: str) -> str:
     """The frontend URL where a user can fill in their per-user env vars."""
     base: Final = os.environ.get("PROXY_BASE_URL", "").rstrip("/")
-    path: Final = f"/ui/?page=mcp-servers&fill_env_vars={quote(server_id, safe='')}"
+    path: Final = f"/ui/mcp-servers?fill_env_vars={quote(server_id, safe='')}"
     return f"{base}{path}" if base else path
 
 

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+2:I[92825,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
+3:I[216370,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js"],"default"]
+4:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
+2:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
+3:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+5:I[27201,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
+0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,11 +1,11 @@
 1:"$Sreact.fragment"
-2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
-3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
-4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
+2:I[363178,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
+3:I[12985,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
+4:I[867271,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[557951,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
+6:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+7:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+8:I[713354,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"912qRXFjlEYHK3EAPXXTc"}