chore: merge litellm_internal_staging into litellm_mistral_voxtral_tts_speech

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,38 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
+        return 0
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/down.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0

.github/e2e-stack/secrets_to_env.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
+
+
+def main() -> int:
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
+        return 1
+    for value in secrets.values():
+        if len(value) >= MIN_MASKED_LENGTH:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -0,0 +1,44 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/workflows/_test-unit-base.yml

@@ -55,17 +55,14 @@ on:
 permissions:
   contents: read
 
+env:
+  UV_PYTHON: "3.12"
+
 jobs:
   run:
-    name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }}
+    name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.job-timeout-minutes }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-    env:
-      UV_PYTHON: ${{ matrix.python-version }}
     permissions:
       contents: read
       pull-requests: read
@@ -88,7 +85,7 @@ jobs:
         timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.UV_PYTHON }}
 
       - name: Set up uv
         if: steps.changes.outputs.decision != 'skip'
@@ -103,9 +100,9 @@ jobs:
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
+          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-
+            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
 
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
@@ -139,7 +136,7 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
           DIST: ${{ inputs.dist }}
-          COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }}
+          COVERAGE_CORE: sysmon
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \
@@ -166,7 +163,7 @@ jobs:
           fi
 
       - name: Save coverage report
-        if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip'
+        if: always() && steps.changes.outputs.decision != 'skip'
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/report-rust-release-wheel.yml

@@ -24,8 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
-      issues: write # PR comments use the issues API
-      pull-requests: read # Current-head validation rejects stale workflow runs
+      pull-requests: write
 
     steps:
       - name: Link release wheel report on PR

.github/workflows/test-code-quality.yml

@@ -74,6 +74,9 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-e2e-changed.yml

@@ -0,0 +1,240 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
+          tests="$(printf '%s\n' "${files}" \
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no changed e2e test files supported by this stack; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          umask 077
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        id: boot
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
+        run: |
+          umask 077
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
+            status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
+            set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
+            echo "pass ${pass} of 3 passed"
+          done
+
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi

.github/workflows/test-terraform-modules.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
   pull_request:
     branches:
@@ -13,6 +14,7 @@ on:
       - "litellm_**"
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
 
 permissions:
@@ -52,3 +54,32 @@ jobs:
       # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
       - name: test
         run: terraform test
+
+  gcp-module:
+    name: fmt, validate, test (gcp)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/gcp
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      - name: test
+        run: terraform test

Dockerfile

@@ -67,6 +67,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -89,6 +90,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 14074
+    "limit": 13429
   },
   "reportArgumentType": {
-    "limit": 2206
+    "limit": 2198
   },
   "reportAssignmentType": {
     "limit": 319
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 4124
+    "limit": 3369
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -48,16 +48,16 @@
     "limit": 30
   },
   "reportInvalidTypeVarUse": {
-    "limit": 2
+    "limit": 1
   },
   "reportMatchNotExhaustive": {
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5601
+    "limit": 5570
   },
   "reportMissingTypeArgument": {
-    "limit": 15284
+    "limit": 15281
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -90,7 +90,7 @@
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 181
+    "limit": 180
   },
   "reportTypedDictNotRequiredAccess": {
     "limit": 22
@@ -105,16 +105,16 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38309
+    "limit": 38271
   },
   "reportUnknownParameterType": {
-    "limit": 19621
+    "limit": 19584
   },
   "reportUnknownVariableType": {
-    "limit": 29844
+    "limit": 29814
   },
   "reportUnnecessaryCast": {
-    "limit": 111
+    "limit": 110
   },
   "reportUnnecessaryComparison": {
     "limit": 687
@@ -123,7 +123,7 @@
     "limit": 4
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 819
+    "limit": 816
   },
   "reportUntypedBaseClass": {
     "limit": 0

ci_cd/generate_model_prices_schema.py

@@ -58,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_audio_formats": {
+        "type": "array",
+        "description": "Audio container formats the model can return.",
+        "items": {"type": "string", "enum": ["mp3", "wav"]},
+    },
     "supported_endpoints": {
         "type": "array",
         "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
@@ -231,6 +236,10 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
         },
         "comment": STRING,
         "audio_transcription_config": STRING,
+        "vertex_ai_audio_api": {
+            "type": "string",
+            "enum": ["lyria_predict", "lyria_interactions"],
+        },
     }
 
 

docker/Dockerfile.database

@@ -65,6 +65,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -87,6 +88,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

docker/Dockerfile.non_root

@@ -71,6 +71,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -99,6 +100,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -109,6 +111,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13; \
     fi
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -433,9 +433,9 @@ _default_detect_secrets_config = {
             "name": "ZendeskSecretKeyDetector",
             "path": _custom_plugins_path + "/zendesk_secret_key.py",
         },
-        {"name": "Base64HighEntropyString", "limit": 3.0},
+        {"name": "Base64HighEntropyString", "limit": 4.5},
         {"name": "HexHighEntropyString", "limit": 3.0},
-    ]
+    ],
 }
 
 
@@ -466,16 +466,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
 
         os.remove(temp_file.name)
 
-        detected_secrets = []
-        for file in secrets.files:
-            for found_secret in secrets[file]:
-                if found_secret.secret_value is None:
-                    continue
-                detected_secrets.append(
-                    {"type": found_secret.type, "value": found_secret.secret_value}
-                )
-
-        return detected_secrets
+        return [
+            {"type": found_secret.type, "value": found_secret.secret_value}
+            for file in sorted(secrets.files)
+            for found_secret in sorted(
+                secrets[file],
+                key=lambda secret: (
+                    -len(secret.secret_value or ""),
+                    secret.type,
+                    secret.secret_value or "",
+                ),
+            )
+            if found_secret.secret_value is not None
+        ]
 
     def redact_text(self, text: str, source: str = "message") -> str:
         """Replace every detected secret in ``text`` with ``[REDACTED]`` and

enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/openai_api_key.py

@@ -3,6 +3,7 @@ This plugin searches for OpenAI API Keys.
 """
 
 import re
+from collections.abc import Generator
 
 from detect_secrets.plugins.base import RegexBasedDetector
 
@@ -16,4 +17,16 @@ class OpenAIApiKeyDetector(RegexBasedDetector):
 
     @property
     def denylist(self) -> list[re.Pattern]:
-        return [re.compile(r"""(sk-[a-zA-Z0-9]{5,})""")]
+        return [
+            re.compile(
+                r"((?:(?<![a-zA-Z0-9])|(?<=%[0-9A-Fa-f]{2}))"
+                r"sk[-_]"
+                r"[a-zA-Z0-9_-]{5,}"
+                r"(?![a-zA-Z0-9_-]))"
+            )
+        ]
+
+    def analyze_string(self, string: str) -> Generator[str, None, None]:
+        # the digit check lives outside the regex: a lookahead re-scans the token
+        # from every `sk` inside it, which is quadratic on `-sk-sk-sk-...` input
+        yield from (match for match in super().analyze_string(string) if re.search(r"[0-9]", match))

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -46,6 +46,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
+    BATCH_CREATE_HIDDEN_PARAM,
     FILE_LIST_CONTINUATION_CHUNK_SIZE,
     MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
@@ -1321,7 +1322,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ## Check if unified_file_id is in the response
             unified_file_id = response._hidden_params.get("unified_file_id")  # managed file id
             unified_batch_id = response._hidden_params.get("unified_batch_id")  # managed batch id
-            is_batch_create: Final = unified_file_id is not None
+            is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
             model_id = cast(Optional[str], response._hidden_params.get("model_id"))
             model_name = cast(Optional[str], response._hidden_params.get("model_name"))
 
@@ -1410,10 +1411,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
                 )
 
-            # Only record batch creation metric on actual create (not retrieve/cancel).
-            # unified_file_id in _hidden_params is only set by the create_batch endpoint.
-            original_unified_file_id = response._hidden_params.get("unified_file_id")
-            if original_unified_file_id:
+            if is_batch_create:
                 prom_logger = self._get_prometheus_logger()
                 if prom_logger:
                     batch_provider = ""

enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
     is_base64_encoded_unified_id,
 )
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.utils import LLMResponseTypes
 from litellm.types.vector_stores import (
     VectorStoreCreateOptionalRequestParams,
     VectorStoreCreateResponse,
@@ -24,6 +25,7 @@ from litellm.types.vector_stores import (
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
+    from litellm.caching.caching import DualCache
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
     from litellm.proxy.utils import PrismaClient as _PrismaClient
 
@@ -156,7 +158,7 @@ class _PROXY_LiteLLMManagedVectorStores(
 
         # Create vector store for each model
         # Convert TypedDict to Dict[str, Any] for base class compatibility
-        request_data_dict: Dict[str, Any] = dict(create_request)
+        request_data_dict: Dict[str, object] = dict(create_request)
         responses = await self.create_resource_for_each_model(
             llm_router=llm_router,
             request_data=request_data_dict,
@@ -209,7 +211,7 @@ class _PROXY_LiteLLMManagedVectorStores(
         limit: Optional[int] = None,
         after: Optional[str] = None,
         order: Optional[str] = None,
-    ) -> Dict[str, Any]:
+    ) -> Dict[str, object]:
         """
         List vector stores created by a user.
         
@@ -301,7 +303,7 @@ class _PROXY_LiteLLMManagedVectorStores(
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
-        cache: Any,
+        cache: "DualCache",
         data: Dict,
         call_type: str,
     ) -> Union[Exception, str, Dict, None]:
@@ -403,8 +405,8 @@ class _PROXY_LiteLLMManagedVectorStores(
         self,
         data: Dict,
         user_api_key_dict: UserAPIKeyAuth,
-        response: Any,
-    ) -> Any:
+        response: LLMResponseTypes,
+    ) -> LLMResponseTypes:
         """
         Post-call hook to transform responses.
         

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.64"
+version = "0.1.65"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.64"
+version = "0.1.65"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -47,6 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -59,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

helm/litellm/templates/migrations-job.yaml

@@ -77,4 +77,16 @@ spec:
       volumes:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.migrationJob.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.migrationJob.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.migrationJob.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/tests/migration_job_tests.yaml

@@ -1,4 +1,4 @@
-suite: test migrations Job ServiceAccount resolution and pod hardening
+suite: test migrations Job ServiceAccount resolution, pod hardening, and scheduling
 templates:
   - migrations-job.yaml
 values:
@@ -188,3 +188,69 @@ tests:
     asserts:
       - notExists:
           path: spec.activeDeadlineSeconds
+
+  - it: renders no scheduling fields by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.tolerations
+      - isNull:
+          path: spec.template.spec.affinity
+
+  - it: renders nodeSelector, tolerations, and affinity from the migrationJob values
+    set:
+      migrationJob.nodeSelector:
+        intent: no-csi-nodes
+      migrationJob.tolerations:
+        - key: intent
+          operator: Equal
+          value: no-csi-nodes
+          effect: NoSchedule
+      migrationJob.affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: intent
+                    operator: In
+                    values:
+                      - no-csi-nodes
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector
+          value:
+            intent: no-csi-nodes
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - key: intent
+              operator: Equal
+              value: no-csi-nodes
+              effect: NoSchedule
+      - equal:
+          path: spec.template.spec.affinity
+          value:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: intent
+                        operator: In
+                        values:
+                          - no-csi-nodes
+
+  - it: does not inherit the gateway's scheduling values
+    set:
+      gateway.nodeSelector:
+        intent: no-csi-nodes
+      gateway.tolerations:
+        - key: intent
+          operator: Equal
+          value: no-csi-nodes
+          effect: NoSchedule
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.tolerations

helm/litellm/values.yaml

@@ -152,6 +152,13 @@ migrationJob:
   # the writable scratch space a read-only root filesystem needs.
   volumes: []
   volumeMounts: []
+  # Scheduling for the Job pod, same shape as gateway.nodeSelector /
+  # gateway.tolerations / gateway.affinity. The Job does not inherit the other
+  # components' scheduling values: a migration usually needs a larger node
+  # than the gateway, so pin it here explicitly.
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
   image:
     repository: ghcr.io/berriai/litellm-migrations
     tag: ""              # defaults to .Chart.AppVersion

litellm-proxy-extras/litellm_proxy_extras/migrations/20260904000000_shadow_eval_model_scope/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "models" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260905120000_add_per_server_oauth_discovery_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "per_server_oauth_discovery" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -343,6 +343,7 @@ model LiteLLM_MCPServerTable {
   delegate_auth_to_upstream Boolean @default(false)
   oauth_passthrough Boolean @default(false)
   dcr_bridge Boolean?
+  per_server_oauth_discovery Boolean @default(false)
   is_byok               Boolean  @default(false)
   byok_description      String[] @default([])
   byok_api_key_help_url String?
@@ -1536,6 +1537,7 @@ model LiteLLM_ShadowEvalJob {
   target_id         String   // hashed virtual key, team_id, or user_id whose traffic this leg shadows
   router_name       String   // first (often only) auto-router under evaluation; router_names is the full set
   router_names      String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
+  models            String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.93"
+version = "0.4.94"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.93"
+version = "0.4.94"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/_lazy_imports.py

@@ -229,7 +229,7 @@ def _module_attribute(module: ModuleType, attr_name: str) -> object:
     return attribute["value"]
 
 
-def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], category: str) -> object:
+def _generic_lazy_import(name: str, import_map: Mapping[str, tuple[str, str]], category: str) -> object:
     """
     Generic function that handles lazy importing for most attributes.
 

litellm/_redis_credential_provider.py

@@ -1,7 +1,7 @@
 import asyncio
 import threading
 import time
-from typing import Any, Final
+from typing import Final, Protocol
 
 from redis.credentials import CredentialProvider
 
@@ -18,6 +18,19 @@ _token_cache: Final[dict[str, tuple[str, float]]] = {}
 _token_cache_lock: Final = threading.Lock()
 
 
+class AzureAccessToken(Protocol):
+    """The ``azure.core.credentials.AccessToken`` shape this module reads."""
+
+    @property
+    def token(self) -> str: ...
+
+
+class AzureCredential(Protocol):
+    """The ``azure-identity`` credential surface this module calls."""
+
+    def get_token(self, *scopes: str) -> AzureAccessToken: ...
+
+
 def _generate_gcp_iam_access_token(service_account: str) -> str:
     """
     Generate GCP IAM access token for Redis authentication.
@@ -115,7 +128,7 @@ class AzureADCredentialProvider(CredentialProvider):
     fail authentication after the initial token expired (~1 hour TTL).
     """
 
-    def __init__(self, credential: Any, username: str | None = None) -> None:
+    def __init__(self, credential: AzureCredential, username: str | None = None) -> None:
         self._credential = credential
         self._username = username
 

litellm/_service_logger.py

@@ -1,7 +1,7 @@
 import asyncio
 from collections.abc import Callable, Coroutine
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 import litellm
 from litellm._logging import verbose_logger
@@ -25,7 +25,30 @@ else:
     UserAPIKeyAuth = Any
 
 
-def _get_otel_v2_class() -> type | None:
+class _ServiceSpanLogger(Protocol):
+    """The OTel logger surface this module drives: the two service-span hooks it calls."""
+
+    async def async_service_success_hook(
+        self,
+        payload: ServiceLoggerPayload,
+        parent_otel_span: Span | None = None,
+        start_time: datetime | float | None = None,
+        end_time: datetime | float | None = None,
+        event_metadata: dict | None = None,
+    ) -> None: ...
+
+    async def async_service_failure_hook(
+        self,
+        payload: ServiceLoggerPayload,
+        error: str | None = "",
+        parent_otel_span: Span | None = None,
+        start_time: datetime | float | None = None,
+        end_time: datetime | float | None = None,
+        event_metadata: dict | None = None,
+    ) -> None: ...
+
+
+def _get_otel_v2_class() -> type[_ServiceSpanLogger] | None:
     """Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent.
 
     Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry
@@ -55,7 +78,7 @@ class ServiceLogging(CustomLogger):
         if "prometheus_system" in litellm.service_callback:
             self.prometheusServicesLogger = PrometheusServicesLogger()
 
-    def _resolve_otel_service_logger(self, callback: Any) -> Any | None:
+    def _resolve_otel_service_logger(self, callback: object) -> _ServiceSpanLogger | None:
         """Resolve the OTel logger (legacy or V2) to emit a service span on.
 
         Returns the logger instance whose ``async_service_*_hook`` should fire for
@@ -70,18 +93,21 @@ class ServiceLogging(CustomLogger):
         """
         otel_v2_cls: Final = _get_otel_v2_class()
 
-        def _is_otel_logger(obj: Any) -> bool:
+        def _as_otel_logger(obj: object) -> _ServiceSpanLogger | None:
             if isinstance(obj, OpenTelemetry):
-                return True
-            return otel_v2_cls is not None and isinstance(obj, otel_v2_cls)
+                return obj
+            if otel_v2_cls is not None and isinstance(obj, otel_v2_cls):
+                return obj
+            return None
 
-        if _is_otel_logger(callback):
-            return callback
+        resolved_callback: Final = _as_otel_logger(callback)
+        if resolved_callback is not None:
+            return resolved_callback
         if callback == "otel":
             from litellm.proxy.proxy_server import open_telemetry_logger
 
-            if open_telemetry_logger is not None and _is_otel_logger(open_telemetry_logger):
-                return open_telemetry_logger
+            if open_telemetry_logger is not None:
+                return _as_otel_logger(open_telemetry_logger)
         return None
 
     @staticmethod

litellm/a2a_protocol/card_resolver.py

@@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
 Extends the A2A SDK's card resolver to support multiple well-known paths.
 """
 
+from collections.abc import Mapping
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
@@ -152,7 +153,7 @@ class LiteLLMA2ACardResolver(_A2ACardResolver):
     async def get_agent_card(
         self,
         relative_card_path: str | None = None,
-        http_kwargs: dict[str, Any] | None = None,
+        http_kwargs: Mapping[str, object] | None = None,
     ) -> "AgentCard":
         """
         Fetch the agent card, trying multiple well-known paths.

litellm/a2a_protocol/providers/bedrock_agentcore/handler.py

@@ -6,8 +6,8 @@ completion bridge that would otherwise strip the envelope.
 """
 
 import json
-from collections.abc import AsyncIterator
-from typing import Any, Final, cast
+from collections.abc import AsyncIterator, Mapping
+from typing import Any, Final
 
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
@@ -28,7 +28,7 @@ class BedrockAgentCoreA2AHandler:
     @staticmethod
     async def handle_non_streaming(
         request_id: str,
-        params: dict[str, Any],
+        params: Mapping[str, object],
         litellm_params: dict[str, Any],
         agent_extra_headers: dict[str, str] | None = None,
     ) -> dict[str, Any]:
@@ -56,7 +56,7 @@ class BedrockAgentCoreA2AHandler:
         verbose_logger.info("BedrockAgentCore A2A: Sending non-streaming request to %s", url)
 
         client: Final = get_async_httpx_client(
-            llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
+            llm_provider=httpxSpecialProvider.A2AProvider,
         )
         response: Final = await client.post(
             url,
@@ -74,7 +74,7 @@ class BedrockAgentCoreA2AHandler:
     @staticmethod
     async def handle_streaming(
         request_id: str,
-        params: dict[str, Any],
+        params: Mapping[str, object],
         litellm_params: dict[str, Any],
         agent_extra_headers: dict[str, str] | None = None,
     ) -> AsyncIterator[dict[str, Any]]:
@@ -103,7 +103,7 @@ class BedrockAgentCoreA2AHandler:
         verbose_logger.info("BedrockAgentCore A2A: Sending streaming request to %s", url)
 
         client: Final = get_async_httpx_client(
-            llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
+            llm_provider=httpxSpecialProvider.A2AProvider,
         )
         response: Final = await client.post(
             url,

litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py

@@ -7,7 +7,7 @@ and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
 
 import json
 from collections.abc import AsyncIterator, Mapping
-from typing import Any, Final
+from typing import Any, Final, Protocol
 
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.litellm_completion_bridge.handler import (
@@ -47,6 +47,12 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
 )
 
 
+class _SSELineSource(Protocol):
+    """Minimal streaming-response surface used to read SSE lines."""
+
+    def aiter_lines(self) -> AsyncIterator[str]: ...
+
+
 def _filter_reserved_headers(
     agent_extra_headers: Mapping[str, str] | None,
 ) -> dict[str, str] | None:
@@ -114,7 +120,7 @@ class BedrockAgentCoreA2ATransformation:
     @staticmethod
     def get_url_and_signed_request(
         request_id: str,
-        params: dict[str, Any],
+        params: Mapping[str, object],
         litellm_params: dict[str, Any],
         method: str = "message/send",
         stream: bool = False,
@@ -213,7 +219,7 @@ class BedrockAgentCoreA2ATransformation:
         return url, signed_headers, signed_body
 
     @staticmethod
-    async def parse_sse_events(response: Any) -> AsyncIterator[dict[str, Any]]:
+    async def parse_sse_events(response: _SSELineSource) -> AsyncIterator[dict[str, Any]]:
         """
         Parse SSE events from an httpx streaming response.
 

litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py

@@ -8,7 +8,7 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
 """
 
 import asyncio
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from typing import Any, Final
 from uuid import uuid4
 
@@ -51,9 +51,9 @@ class WatsonxOrchestrateTransformation:
         wxo_agent_id: str,
         text: str,
         thread_id: str | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """Build the WXO POST /v1/orchestrate/runs request body."""
-        body: Final[dict[str, Any]] = {
+        body: Final[dict[str, object]] = {
             "agent_id": wxo_agent_id,
             "message": {
                 "role": "user",
@@ -70,7 +70,7 @@ class WatsonxOrchestrateTransformation:
         return body
 
     @staticmethod
-    def extract_text_from_wxo_result(result: Any) -> str:
+    def extract_text_from_wxo_result(result: object) -> str:
         """
         Extract response text from a WXO run result.
 
@@ -103,7 +103,7 @@ class WatsonxOrchestrateTransformation:
         return ""
 
     @staticmethod
-    def extract_text_from_a2a_message_response(a2a_response: dict[str, Any]) -> str:
+    def extract_text_from_a2a_message_response(a2a_response: Mapping[str, object]) -> str:
         result: Final = a2a_response.get("result")
         if not isinstance(result, dict):
             verbose_logger.warning("WXO: A2A response missing result object")
@@ -119,7 +119,7 @@ class WatsonxOrchestrateTransformation:
         return ""
 
     @staticmethod
-    def build_a2a_message_response(request_id: str, text: str) -> dict[str, Any]:
+    def build_a2a_message_response(request_id: str, text: str) -> dict[str, object]:
         """
         Build a standard A2A non-streaming SendMessageResponse (kind=message).
         """
@@ -140,7 +140,7 @@ class WatsonxOrchestrateTransformation:
         request_id: str,
         chunk_size: int = 50,
         delay_ms: int = 10,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         """
         Emit standard A2A streaming events from a completed text response.
 

litellm/a2a_protocol/streaming_iterator.py

@@ -148,9 +148,9 @@ class A2AStreamingIterator:
         except Exception as e:
             verbose_logger.debug("Error in A2A streaming completion handler: %s", e)
 
-    def _build_logging_result(self, usage: litellm.Usage) -> dict[str, Any]:
+    def _build_logging_result(self, usage: litellm.Usage) -> dict[str, object]:
         """Build a result dict for logging."""
-        result: Final[dict[str, Any]] = {
+        result: Final[dict[str, object]] = {
             "id": getattr(self.request, "id", "unknown"),
             "jsonrpc": "2.0",
             "usage": (usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)),

litellm/a2a_protocol/utils.py

@@ -48,7 +48,7 @@ class A2ARequestUtils:
         return " ".join(text_parts)
 
     @staticmethod
-    def extract_text_from_response(response_dict: dict[str, Any]) -> str:
+    def extract_text_from_response(response_dict: Mapping[str, object]) -> str:
         """
         Extract text content from A2A response result.
 
@@ -111,7 +111,7 @@ class A2ARequestUtils:
     @staticmethod
     def calculate_usage_from_request_response(
         request: "SendMessageRequest | SendStreamingMessageRequest",
-        response_dict: dict[str, Any],
+        response_dict: Mapping[str, object],
     ) -> tuple[int, int, int]:
         """
         Calculate token usage from A2A request and response.
@@ -170,5 +170,5 @@ def extract_text_from_a2a_message(message: Any) -> str:
     return A2ARequestUtils.extract_text_from_message(message)
 
 
-def extract_text_from_a2a_response(response_dict: dict[str, Any]) -> str:
+def extract_text_from_a2a_response(response_dict: Mapping[str, object]) -> str:
     return A2ARequestUtils.extract_text_from_response(response_dict)

litellm/assistants/utils.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping, Sequence
 from typing import Final
 
 import litellm
@@ -10,20 +11,22 @@ def get_optional_params_add_message(
     role: str | None,
     content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
     attachments: list[Attachment] | None,
-    metadata: dict | None,
+    metadata: Mapping[str, object] | None,
     custom_llm_provider: str,
-    **kwargs,
-):
+    **kwargs: object,
+) -> dict[str, object]:
     """
     Azure doesn't support 'attachments' for creating a message
 
     Reference - https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message
     """
-    passed_params: Final = locals()
-    custom_llm_provider = passed_params.pop("custom_llm_provider")
-    special_params: Final = passed_params.pop("kwargs")
-    for k, v in special_params.items():
-        passed_params[k] = v
+    passed_params: Final[Mapping[str, object]] = {
+        "role": role,
+        "content": content,
+        "attachments": attachments,
+        "metadata": metadata,
+        **kwargs,
+    }
 
     default_params: Final = {
         "role": None,
@@ -33,10 +36,10 @@ def get_optional_params_add_message(
     }
 
     non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
-    optional_params = {}
+    optional_params: dict[str, object] = {}
 
     ## raise exception if non-default value passed for non-openai/azure embedding calls
-    def _check_valid_arg(supported_params):
+    def _check_valid_arg(supported_params: Sequence[str]) -> Mapping[str, object] | None:
         if len(non_default_params.keys()) > 0:
             keys: Final = list(non_default_params.keys())
             for k in keys:
@@ -71,14 +74,18 @@ def get_optional_params_image_gen(
     style: str | None = None,
     user: str | None = None,
     custom_llm_provider: str | None = None,
-    **kwargs,
-):
+    **kwargs: object,
+) -> dict[str, object]:
     # retrieve all parameters passed to the function
-    passed_params: Final = locals()
-    custom_llm_provider = passed_params.pop("custom_llm_provider")
-    special_params: Final = passed_params.pop("kwargs")
-    for k, v in special_params.items():
-        passed_params[k] = v
+    passed_params: Final[Mapping[str, object]] = {
+        "n": n,
+        "quality": quality,
+        "response_format": response_format,
+        "size": size,
+        "style": style,
+        "user": user,
+        **kwargs,
+    }
 
     default_params: Final = {
         "n": None,
@@ -90,10 +97,10 @@ def get_optional_params_image_gen(
     }
 
     non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
-    optional_params = {}
+    optional_params: dict[str, object] = {}
 
     ## raise exception if non-default value passed for non-openai/azure embedding calls
-    def _check_valid_arg(supported_params):
+    def _check_valid_arg(supported_params: Sequence[str]) -> Mapping[str, object] | None:
         if len(non_default_params.keys()) > 0:
             keys: Final = list(non_default_params.keys())
             for k in keys:

litellm/batches/batch_utils.py

@@ -25,6 +25,27 @@ class BatchCostUsageResult:
     failed_requests: int
 
 
+_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
+_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
+
+
+def batch_cost_is_final(batch: Batch) -> bool:
+    """Whether this retrieve of the batch is the one to account its cost from.
+
+    A batch still in flight has nothing to price, and a "completed" batch can report
+    no output_file_id for a moment before the output populates; pricing either records
+    $0 under the batch's single spend row and pins it there. Final means a completed
+    batch whose output file has arrived or whose counts prove no line succeeded, or
+    any other terminal status (failed, cancelled, expired).
+    """
+    if batch.status not in _TERMINAL_BATCH_STATUSES:
+        return False
+    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
+        return True
+    request_counts: Final = batch.request_counts
+    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
+
+
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: list[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
@@ -160,7 +181,7 @@ def _classify_output_line_stats(
 
 
 def _safe_output_line_stats(
-    entry: Mapping[str, Any],
+    entry: Mapping[str, object],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
     model_info: ModelInfo | None,
@@ -182,7 +203,7 @@ def _safe_output_line_stats(
 
 
 def _compute_output_line_stats(
-    entry: Mapping[str, Any],
+    entry: Mapping[str, object],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
     model_info: ModelInfo | None,
@@ -213,7 +234,7 @@ def _compute_output_line_stats(
 
 
 def _output_line_cost(
-    response_body: Mapping[str, Any],
+    response_body: Mapping[str, object],
     usage: Usage,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
@@ -556,7 +577,7 @@ def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict]:
 
 def _parse_batch_output_line(line: bytes) -> dict | None:
     try:
-        parsed: Final = json.loads(line)
+        parsed: Final[object] = json.loads(line)
     except ValueError as e:
         verbose_logger.warning("skipping malformed batch output line: %s", str(e))
         return None
@@ -601,7 +622,7 @@ def _count_entry_tokens(
     return 0
 
 
-def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
+def _count_prompt_or_input_tokens(model: str, value: object) -> int:
     """Token-count a ``prompt`` / ``input`` field that the OpenAI batch
     schema allows in four shapes:
 
@@ -680,7 +701,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
 
 def _get_response_from_batch_job_output_file(
     batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
-) -> Mapping[str, Any]:
+) -> Mapping[str, object]:
     """
     Get the response from the batch job output file
     """

litellm/caching/caching_handler.py

@@ -672,7 +672,7 @@ class LLMCachingHandler:
     def _async_log_cache_hit_on_callbacks(
         self,
         logging_obj: LiteLLMLoggingObj,
-        cached_result: Any,
+        cached_result: object,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
         cache_hit: bool,
@@ -1184,7 +1184,7 @@ class LLMCachingHandler:
         logging_obj: LiteLLMLoggingObj,
         model: str,
         kwargs: dict[str, Any],
-        cached_result: Any,
+        cached_result: object,
         is_async: bool,
         is_embedding: bool = False,
         custom_llm_provider: str | None = None,

litellm/caching/dual_cache.py

@@ -257,7 +257,7 @@ class DualCache(BaseCache):
         self,
         current_time: float,
         keys: list[str],
-        result: Sequence[Any],
+        result: Sequence[object],
     ) -> tuple[list[str], dict[str, float | None]]:
         """
         Atomically choose keys to fetch from Redis and reserve their access time.

litellm/caching/redis_semantic_cache.py

@@ -116,7 +116,7 @@ class RedisSemanticCache(BaseCache):
                 password = password or os.environ["REDIS_PASSWORD"]
             except KeyError as e:
                 # Raise a more informative exception if any of the required keys are missing
-                missing_var: Final = e.args[0]
+                missing_var: Final[object] = e.args[0]
                 raise ValueError(
                     f"Missing required Redis configuration: {missing_var}. Provide {missing_var} or redis_url."
                 ) from e
@@ -273,7 +273,7 @@ class RedisSemanticCache(BaseCache):
         return prompt or None
 
     @classmethod
-    def _collect_responses_input_text(cls, value: Any, prompt_parts: list[str]) -> None:
+    def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None:
         value = cls._coerce_response_input_value(value)
         if value is None:
             return
@@ -334,7 +334,7 @@ class RedisSemanticCache(BaseCache):
             resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
         )
 
-    def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
+    def _get_embedding(self, prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
         """
         Routes through the proxy Router when the embedding model is a Router
         deployment so per-deployment auth (e.g. Bedrock aws_role_name) applies,
@@ -425,7 +425,7 @@ class RedisSemanticCache(BaseCache):
 
             prompt_embedding: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
 
-            store_kwargs: Final[dict[str, Any]] = {
+            store_kwargs: Final[dict[str, object]] = {
                 "vector": prompt_embedding,
                 "filters": self._get_cache_filters(key),
             }
@@ -504,7 +504,7 @@ class RedisSemanticCache(BaseCache):
             print_verbose(f"Error retrieving from Redis semantic cache: {e}")
             kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
 
-    async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
+    async def _get_async_embedding(self, prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
         """
         Asynchronously generate an embedding for the given prompt.
 
@@ -571,7 +571,7 @@ class RedisSemanticCache(BaseCache):
             # Generate embedding for the value (response) to cache
             prompt_embedding: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
 
-            store_kwargs: Final[dict[str, Any]] = {
+            store_kwargs: Final[dict[str, object]] = {
                 "vector": prompt_embedding,
                 "filters": self._get_cache_filters(key),
             }
@@ -665,7 +665,7 @@ class RedisSemanticCache(BaseCache):
         aindex: Final = await self.llmcache._get_async_index()
         return await aindex.info()
 
-    async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: object) -> None:
+    async def async_set_cache_pipeline(self, cache_list: list[tuple[str, object]], **kwargs: object) -> None:
         """
         Asynchronously store multiple values in the semantic cache.
 

litellm/compression/compress.py

@@ -66,7 +66,7 @@ def _build_retrieval_tools(keys: list[str], call_type: str) -> list[dict]:
     return cast(list[dict], anthropic_tools)
 
 
-def _content_to_text(content: Any) -> str:
+def _content_to_text(content: object) -> str:
     """
     Convert OpenAI/Anthropic message content blocks to plain text.
 
@@ -78,7 +78,7 @@ def _content_to_text(content: Any) -> str:
     Implemented iteratively (stack-based) to avoid unbounded recursion.
     """
     parts: Final[list[str]] = []
-    stack: Final[list[Any]] = [content]
+    stack: Final[list[object]] = [content]
     while stack:
         item = stack.pop()
         if isinstance(item, str):
@@ -111,7 +111,7 @@ def _normalize_messages_for_compression(
             f"Unsupported call_type={call_type!r} for compression. Expected one of: {sorted(_SUPPORTED_CALL_TYPES)}."
         )
 
-    original_messages: Final[list[dict[str, Any]]] = [dict(m) for m in messages]
+    original_messages: Final[list[dict[str, object]]] = [dict(m) for m in messages]
 
     normalized_messages: Final[list[dict]] = []
     for msg in original_messages:
@@ -132,7 +132,7 @@ def _extract_last_user_message(messages: list[dict]) -> str:
     return ""
 
 
-def _extract_tool_use_ids(content: Any) -> list[str]:
+def _extract_tool_use_ids(content: object) -> list[str]:
     if not isinstance(content, list):
         return []
     tool_use_ids: Final[list[str]] = []
@@ -147,7 +147,7 @@ def _extract_tool_use_ids(content: Any) -> list[str]:
     return tool_use_ids
 
 
-def _extract_tool_result_ids(content: Any) -> set[str]:
+def _extract_tool_result_ids(content: object) -> set[str]:
     if not isinstance(content, list):
         return set()
     tool_result_ids: Final[set[str]] = set()
@@ -337,7 +337,7 @@ def compress(
     compression_trigger: int = 200_000,
     compression_target: int | None = None,
     embedding_model: str | None = None,
-    embedding_model_params: dict[str, Any] | None = None,
+    embedding_model_params: Mapping[str, object] | None = None,
     compression_cache: DualCache | None = None,
 ) -> CompressedResult:
     """

litellm/compression/scoring/embedding_scorer.py

@@ -5,6 +5,7 @@ Computes cosine similarity between the query embedding and each message embeddin
 """
 
 import math
+from collections.abc import Mapping
 from typing import Any, Final
 
 from litellm.caching.dual_cache import DualCache
@@ -49,7 +50,7 @@ def embedding_score_messages(
     messages: list[dict],
     model: str,
     cache: DualCache | None = None,
-    embedding_model_params: dict[str, Any] | None = None,
+    embedding_model_params: Mapping[str, object] | None = None,
 ) -> list[float]:
     """
     Score each message's semantic similarity to the query using embeddings.

litellm/constants.py

@@ -40,7 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
         "router_general_settings",
         "ignore_invalid_deployments",
         "fallback_access_check",
-        "heuristic_v2_router_limit",
+        "auto_router_capability_limit",
     }
 )
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
@@ -89,6 +89,7 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Data URIs exceeding this are replaced with a size placeholder.
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
+BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
 REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
 # in-memory stand-in handed to provider converters for redacted arguments; never stored
 REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@@ -215,6 +216,9 @@ MAX_CALLBACKS: Final = get_env_int("LITELLM_MAX_CALLBACKS", 100)
 # so the deployment-level hook does not re-run them for the same request
 PRE_CALL_EXECUTED_GUARDRAILS_KEY: Final = "_pre_call_executed_guardrails"
 
+# Attribute stamped on log_guardrail_information wrappers so __init_subclass__ does not wrap them again
+LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information"
+
 # Generic fallback for unknown models
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)

litellm/containers/endpoint_factory.py

@@ -11,7 +11,7 @@ import json
 from collections.abc import Callable
 from functools import partial
 from pathlib import Path
-from typing import Any, Final, Literal
+from typing import Final, Literal
 
 import litellm
 from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
@@ -56,9 +56,9 @@ def create_sync_endpoint_function(endpoint_config: dict) -> Callable:
     def endpoint_func(
         timeout: int = 600,
         custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-        extra_headers: dict[str, Any] | None = None,
-        extra_query: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_headers: dict[str, object] | None = None,
+        extra_query: dict[str, object] | None = None,
+        extra_body: dict[str, object] | None = None,
         **kwargs,
     ):
         local_vars: Final = locals()
@@ -145,9 +145,9 @@ def create_async_endpoint_function(
     async def async_endpoint_func(
         timeout: int = 600,
         custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-        extra_headers: dict[str, Any] | None = None,
-        extra_query: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_headers: dict[str, object] | None = None,
+        extra_query: dict[str, object] | None = None,
+        extra_body: dict[str, object] | None = None,
         **kwargs,
     ):
         local_vars: Final = locals()

litellm/cost_calculator.py

@@ -81,6 +81,7 @@ from litellm.llms.together_ai.cost_calculator import (
     get_model_params_and_category,
     has_together_registry_pricing,
 )
+from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -496,6 +497,13 @@ def cost_per_token(
 
     # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
     if call_type == "speech" or call_type == "aspeech":
+        lyria_generation_cost: Final = (
+            get_vertex_ai_lyria_generation_cost(model=model_without_prefix)
+            if custom_llm_provider in ("vertex_ai", "vertex_ai_beta")
+            else None
+        )
+        if lyria_generation_cost is not None:
+            return 0.0, lyria_generation_cost
         speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider)
         cost_metric: Final = select_cost_metric_for_model(speech_model_info)
         prompt_cost: float = 0.0

litellm/exceptions.py

@@ -85,7 +85,7 @@ _RATE_LIMIT_CATEGORY_VALUES: Final = frozenset(c.value for c in RateLimitErrorCa
 _RATE_LIMIT_TYPE_VALUES: Final = frozenset(t.value for t in RateLimitType)
 
 
-def validate_rate_limit_category(value: Any) -> str | None:
+def validate_rate_limit_category(value: object) -> str | None:
     """Return ``value`` only if it matches a known :class:`RateLimitErrorCategory`.
 
     Used at duck-typed read sites (StandardLoggingPayload extraction, Prometheus
@@ -100,7 +100,7 @@ def validate_rate_limit_category(value: Any) -> str | None:
     return None
 
 
-def validate_rate_limit_type(value: Any) -> str | None:
+def validate_rate_limit_type(value: object) -> str | None:
     """Return ``value`` only if it matches a known :class:`RateLimitType`.
 
     See :func:`validate_rate_limit_category` for the rationale.

litellm/experimental_mcp_client/client.py

@@ -6,17 +6,35 @@ import asyncio
 import base64
 import os
 from collections.abc import Awaitable, Callable, Generator
+from contextlib import AbstractAsyncContextManager
 from datetime import timedelta
 from functools import partial
 from importlib import metadata
-from typing import Any, Final, TypeVar
+from typing import Any, Final, Protocol, TypeAlias, TypeVar
 
 import httpx
+from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
 from mcp import ClientSession, McpError, ReadResourceResult, Resource, StdioServerParameters
 from mcp.client.sse import sse_client
 from mcp.client.stdio import stdio_client
+from mcp.shared.message import SessionMessage
+from typing_extensions import Unpack
 
-streamable_http_client: Any | None = None
+_TransportStreams: TypeAlias = tuple[
+    MemoryObjectReceiveStream[SessionMessage | Exception],
+    MemoryObjectSendStream[SessionMessage],
+    Unpack[tuple[object, ...]],
+]
+_TransportContext: TypeAlias = AbstractAsyncContextManager[_TransportStreams]
+
+
+class _StreamableHttpClientFactory(Protocol):
+    """The ``streamable_http_client`` entry point this module calls on the installed MCP SDK."""
+
+    def __call__(self, *, url: str, http_client: httpx.AsyncClient | None) -> _TransportContext: ...
+
+
+streamable_http_client: _StreamableHttpClientFactory | None = None
 try:
     import mcp.client.streamable_http as streamable_http_module
 
@@ -216,10 +234,12 @@ class MCPSigV4Auth(httpx.Auth):
         aws_region_name: str,
     ):
         """Call STS AssumeRole and return temporary credentials."""
+        import time
+
         import boto3
         from botocore.credentials import Credentials
 
-        session_name: Final = aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
+        session_name: Final = aws_session_name or f"litellm-mcp-{int(time.time())}"
         sts_kwargs: Final[dict] = {"region_name": aws_region_name}
         if aws_access_key_id and aws_secret_access_key:
             sts_kwargs["aws_access_key_id"] = aws_access_key_id
@@ -315,7 +335,7 @@ class MCPClient:
 
     def _create_transport_context(
         self,
-    ) -> tuple[Any, httpx.AsyncClient | None]:
+    ) -> tuple[_TransportContext, httpx.AsyncClient | None]:
         """
         Create the appropriate transport context based on transport type.
         Returns:
@@ -408,7 +428,7 @@ class MCPClient:
 
     async def _execute_session_operation(
         self,
-        transport_ctx: Any,
+        transport_ctx: _TransportContext,
         operation: Callable[[ClientSession], Awaitable[TSessionResult]],
     ) -> TSessionResult:
         """

litellm/files/main.py

@@ -14,6 +14,7 @@ from functools import partial
 from typing import Any, Final, Literal, cast
 
 import httpx
+from openai import AsyncOpenAI, OpenAI
 
 # Type aliases for provider parameters
 FileCreateProvider = Literal[
@@ -431,7 +432,7 @@ async def afile_delete(
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
     **kwargs,
-) -> Coroutine[Any, Any, FileObject]:
+) -> Coroutine[object, object, FileObject]:
     """
     Async: Delete file
 
@@ -1002,8 +1003,8 @@ def file_content_streaming(
     timeout: float | httpx.Timeout,
     logging_obj: LiteLLMLoggingObj | None,
     _is_async: bool,
-    client: Any | None,
-) -> FileContentStreamingResult | Coroutine[Any, Any, FileContentStreamingResult]:
+    client: OpenAI | AsyncOpenAI | None,
+) -> FileContentStreamingResult | Coroutine[object, object, FileContentStreamingResult]:
     if logging_obj is not None:
         logging_obj.model = model or ""
         logging_obj.model_call_details["model"] = model or ""
@@ -1028,8 +1029,8 @@ def file_content_streaming(
             headers=response.headers,
         )
 
-    response: FileContentStreamingResult | Coroutine[Any, Any, FileContentStreamingResult] = FileContentStreamingResult(
-        stream_iterator=iter(()), headers={}
+    response: FileContentStreamingResult | Coroutine[object, object, FileContentStreamingResult] = (
+        FileContentStreamingResult(stream_iterator=iter(()), headers={})
     )
     if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
         openai_creds: Final = get_openai_credentials(

litellm/fine_tuning/main.py

@@ -11,7 +11,7 @@ https://platform.openai.com/docs/api-reference/fine-tuning
 import asyncio
 import contextvars
 import os
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from functools import partial
 from typing import Any, Final, Literal
 
@@ -37,8 +37,8 @@ vertex_fine_tuning_apis_instance: Final = VertexFineTuningAPI()
 
 def _prepare_azure_extra_body(
     extra_body: dict[str, Any] | None,
-    kwargs: dict[str, Any],
-    azure_specific_hyperparams: dict[str, Any],
+    kwargs: Mapping[str, object],
+    azure_specific_hyperparams: Mapping[str, object],
 ) -> dict[str, Any]:
     """
     Prepare extra_body for Azure fine-tuning API by combining Azure-specific parameters.
@@ -138,7 +138,7 @@ def _build_fine_tuning_job_data(model, training_file, hyperparameters, suffix, v
 
 
 def _resolve_fine_tuning_timeout(
-    timeout: Any,
+    timeout: float | str | httpx.Timeout | None,
     custom_llm_provider: str,
 ) -> float | httpx.Timeout:
     """Normalise a raw timeout value to a float (seconds) or httpx.Timeout for fine-tuning calls."""
@@ -163,7 +163,7 @@ def create_fine_tuning_job(
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
     **kwargs,
-) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
+) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
     """
     Creates a fine-tuning job which begins the process of creating a new model from a given dataset.
 
@@ -375,7 +375,7 @@ def cancel_fine_tuning_job(
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
     **kwargs,
-) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
+) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
     """
     Immediately cancel a fine-tune job.
 
@@ -682,7 +682,7 @@ def retrieve_fine_tuning_job(
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
     **kwargs,
-) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
+) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
     """
     Get info about a fine-tuning job.
     """

litellm/images/utils.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from io import BufferedReader, BytesIO
 from typing import Any, Final, cast, get_type_hints
 
@@ -61,7 +62,7 @@ class ImageEditRequestUtils:
 
     @staticmethod
     def get_requested_image_edit_optional_param(
-        params: dict[str, Any],
+        params: Mapping[str, object],
     ) -> ImageEditOptionalRequestParams:
         """
         Filter parameters to only include those defined in ImageEditOptionalRequestParams.

litellm/integrations/arize/_utils.py

@@ -2,7 +2,7 @@ import json
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
-from typing_extensions import override
+from typing_extensions import ReadOnly, TypedDict, override
 
 from litellm._logging import verbose_logger
 from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import (
@@ -492,12 +492,12 @@ def _sanitize_optional_params(optional_params: dict | None) -> dict:
     return optional_params
 
 
-def _set_metadata_attributes(span: "Span", metadata: Any | None, span_attrs) -> None:
+def _set_metadata_attributes(span: "Span", metadata: object | None, span_attrs) -> None:
     if metadata is not None:
         safe_set_attribute(span, span_attrs.METADATA, safe_dumps(metadata))
 
 
-def _extract_metadata_tools(metadata: Any | None) -> list | None:
+def _extract_metadata_tools(metadata: object | None) -> list | None:
     if not isinstance(metadata, dict):
         return None
     llm_obj: Final = metadata.get("llm")
@@ -670,7 +670,22 @@ def _get_tool_calls(message) -> list | None:
     return tool_calls if isinstance(tool_calls, list) and tool_calls else None
 
 
-def _normalize_tool_call(raw_tc) -> dict[str, Any] | None:
+class _NormalizedToolCallFunction(TypedDict):
+    """The ``function`` sub-object of a normalized tool call."""
+
+    name: ReadOnly[object]
+    arguments: ReadOnly[object]
+
+
+class _NormalizedToolCall(TypedDict):
+    """A tool call reduced to the stable shape the OpenInference emitters read."""
+
+    id: ReadOnly[object]
+    type: ReadOnly[object]
+    function: ReadOnly[_NormalizedToolCallFunction]
+
+
+def _normalize_tool_call(raw_tc) -> _NormalizedToolCall | None:
     """Normalize a single tool_call (dict or Pydantic) into a stable shape:
 
         {"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}}

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -16,23 +16,32 @@ import asyncio
 import os
 import time
 import traceback
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeVar
 from urllib.parse import urlparse
 
 from litellm._logging import verbose_logger
+from litellm.integrations.batch_utils import (
+    BatchSendCancelled,
+    send_batch_with_413_split,
+    undelivered_after_http_error,
+)
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
+    MaskedHTTPStatusError,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.types.integrations.azure_sentinel import AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
 from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 DEFAULT_AZURE_AUTHORITY_HOST: Final = "https://login.microsoftonline.com"
 DEFAULT_AZURE_MONITOR_SCOPE: Final = "https://monitor.azure.com/.default"
 
+_QueuedPayload = TypeVar("_QueuedPayload", StandardLoggingPayload, StandardAuditLogPayload)
+
 MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
     {
         "login.microsoftonline.com": DEFAULT_AZURE_MONITOR_SCOPE,
@@ -153,6 +162,8 @@ class AzureSentinelLogger(CustomBatchLogger):
         asyncio.create_task(self.periodic_flush())
         self.log_queue: list[StandardLoggingPayload] = []
         self.audit_log_queue: list[StandardAuditLogPayload] = []
+        self.logs_awaiting_retry = False
+        self.audit_logs_awaiting_retry = False
 
     @staticmethod
     def _normalize_authority_host(authority_host: str) -> str:
@@ -245,8 +256,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -275,8 +286,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -298,12 +309,24 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.audit_log_queue.append(audit_log)
 
-            if len(self.audit_log_queue) >= self.batch_size:
-                await self.async_send_audit_batch()
+            if len(self.audit_log_queue) >= self.batch_size and not self.audit_logs_awaiting_retry:
+                await self._threshold_send_audit_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Audit Log Layer Error - %s\n%s", e, traceback.format_exc())
 
+    async def _threshold_send_logs(self) -> None:
+        async with self.flush_lock:
+            if self.logs_awaiting_retry or len(self.log_queue) < self.batch_size:
+                return
+            await self.async_send_batch()
+
+    async def _threshold_send_audit_logs(self) -> None:
+        async with self.flush_lock:
+            if self.audit_logs_awaiting_retry or len(self.audit_log_queue) < self.batch_size:
+                return
+            await self.async_send_audit_batch()
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -311,67 +334,110 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.log_queue,
-            api_endpoint=self.api_endpoint,
-            log_type="logs",
-        )
+        batch_to_send: Final = tuple(self.log_queue)
+        self.log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.api_endpoint,
+                log_type="logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.log_queue = self._requeue(cancelled.undelivered, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.log_queue = self._requeue(batch_to_send, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise
+        self.log_queue = self._requeue(undelivered, self.log_queue, "logs")
+        self.logs_awaiting_retry = bool(undelivered) and bool(self.log_queue)
 
     async def async_send_audit_batch(self):
         """
         Sends the batch of audit logs to Azure Monitor Logs Ingestion API
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.audit_log_queue,
-            api_endpoint=self.audit_api_endpoint,
-            log_type="audit logs",
+        batch_to_send: Final = tuple(self.audit_log_queue)
+        self.audit_log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.audit_api_endpoint,
+                log_type="audit logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.audit_log_queue = self._requeue(cancelled.undelivered, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.audit_log_queue = self._requeue(batch_to_send, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise
+        self.audit_log_queue = self._requeue(undelivered, self.audit_log_queue, "audit logs")
+        self.audit_logs_awaiting_retry = bool(undelivered) and bool(self.audit_log_queue)
+
+    def _requeue(
+        self,
+        undelivered: tuple[_QueuedPayload, ...],
+        queue: list[_QueuedPayload],
+        log_type: str,
+    ) -> list[_QueuedPayload]:
+        merged: Final = [*undelivered, *queue]  # mutable-ok: queue trimming returns a mutable logger queue
+        overflow: Final = len(merged) - self.max_queue_size
+        if overflow <= 0:
+            return merged
+
+        verbose_logger.warning(
+            "Azure Sentinel: %s queue exceeded max_queue_size=%s, dropped %s oldest records",
+            log_type,
+            self.max_queue_size,
+            overflow,
         )
+        return merged[overflow:]
 
     async def _async_send_batch_to_api(
         self,
-        log_queue: list[StandardLoggingPayload | StandardAuditLogPayload],
+        log_queue: tuple[_QueuedPayload, ...],
         api_endpoint: str,
         log_type: str,
-    ) -> None:
+    ) -> tuple[_QueuedPayload, ...]:
+        if not log_queue:
+            return ()
+
+        verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
         try:
-            if not log_queue:
-                return
-
-            verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
-
-            # Get OAuth2 token
             bearer_token: Final = await self._get_oauth_token()
+        except MaskedHTTPStatusError as e:
+            return undelivered_after_http_error(log_queue, e.status_code, "Azure Sentinel OAuth token", str(e))
+        except Exception as e:
+            verbose_logger.exception("Azure Sentinel Error getting OAuth token - %s", e)
+            return tuple(log_queue)
 
-            # Convert log queue to JSON array format expected by Logs Ingestion API
-            # Each log entry should be a JSON object in the array
-            body: Final = safe_dumps(log_queue)
+        headers: Final = {
+            "Authorization": f"Bearer {bearer_token}",
+            "Content-Type": "application/json",
+        }
 
-            # Set headers for Logs Ingestion API
-            headers: Final = {
-                "Authorization": f"Bearer {bearer_token}",
-                "Content-Type": "application/json",
-            }
-
-            # Send the request
-            response = await self.async_httpx_client.post(url=api_endpoint, data=body.encode("utf-8"), headers=headers)
-
-            if response.status_code not in [200, 204]:
-                verbose_logger.error(
-                    "Azure Sentinel API error: status_code=%s, response=%s",
-                    response.status_code,
-                    response.text,
-                )
-                raise Exception(f"Failed to send logs to Azure Sentinel: {response.status_code} - {response.text}")
-
-            verbose_logger.debug(
-                "Azure Sentinel: Response from API status_code: %s",
-                response.status_code,
+        async def _send_batch(batch: Sequence[_QueuedPayload]):
+            body: Final = safe_dumps(batch)
+            return await self.async_httpx_client.post(
+                url=api_endpoint,
+                data=body.encode("utf-8"),
+                headers=headers,
             )
 
-        except Exception as e:
-            verbose_logger.exception("Azure Sentinel Error sending batch API - %s\n%s", e, traceback.format_exc())
-        finally:
-            log_queue.clear()
+        return await send_batch_with_413_split(
+            batch=log_queue,
+            send_batch=_send_batch,
+            exceeds_limits=lambda batch: (
+                len(batch) > self.batch_size
+                or len(safe_dumps(batch).encode("utf-8")) > AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
+            ),
+            success_status_codes=frozenset({200, 204}),
+            integration_name="Azure Sentinel",
+            drop_error_message="Azure Sentinel API Error - Payload too large for a single record",
+            non_success_handler=undelivered_after_http_error,
+        )
 
     async def flush_queue(self):
         if self.flush_lock is None:

litellm/integrations/batch_utils.py

@@ -0,0 +1,160 @@
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from typing import Final, Generic, TypeVar
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
+
+_BatchItem = TypeVar("_BatchItem")
+
+_RETRYABLE_CLIENT_STATUS_CODES: Final = frozenset({408, 429})
+
+
+def is_retryable_status(status_code: int) -> bool:
+    return not 400 <= status_code < 500 or status_code in _RETRYABLE_CLIENT_STATUS_CODES
+
+
+def undelivered_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    """The records to requeue after a non-2xx: all of them on a status a retry can clear, none on
+    a 4xx that would only repeat, since retaining those retries a misconfiguration forever."""
+    if is_retryable_status(status_code):
+        verbose_logger.error(
+            "%s API error: status_code=%s, will retry %s records - %s",
+            integration_name,
+            status_code,
+            len(batch),
+            detail,
+        )
+        return tuple(batch)
+    verbose_logger.error(
+        "%s API error: status_code=%s is not retryable, dropped %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return ()
+
+
+def requeue_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    verbose_logger.error(
+        "%s API error: status_code=%s, will retry %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return tuple(batch)
+
+
+class BatchSendCancelled(asyncio.CancelledError, Generic[_BatchItem]):
+    """Cancellation of a batch send, carrying only the records the destination never accepted.
+
+    A batch split under the size cap is delivered in pieces, so requeueing all of it after a
+    cancellation partway through would send the accepted pieces a second time.
+    """
+
+    def __init__(self, undelivered: tuple[_BatchItem, ...]) -> None:
+        super().__init__()
+        self.undelivered: Final = undelivered
+
+
+async def _keep_the_remainder_on_cancel(
+    send: Awaitable[tuple[_BatchItem, ...]],
+    remainder: Sequence[_BatchItem],
+) -> tuple[_BatchItem, ...]:
+    try:
+        return await send
+    except BatchSendCancelled as cancelled:
+        raise BatchSendCancelled((*cancelled.undelivered, *remainder)) from cancelled
+
+
+async def send_batch_with_413_split(
+    batch: Sequence[_BatchItem],
+    send_batch: Callable[[Sequence[_BatchItem]], Awaitable[httpx.Response]],
+    exceeds_limits: Callable[[Sequence[_BatchItem]], bool],
+    success_status_codes: frozenset[int],
+    integration_name: str,
+    drop_error_message: str,
+    non_success_handler: Callable[
+        [Sequence[_BatchItem], int, str, str], tuple[_BatchItem, ...]
+    ] = requeue_after_http_error,
+) -> tuple[_BatchItem, ...]:
+    async def _halve() -> tuple[_BatchItem, ...]:
+        midpoint: Final = len(batch) // 2
+        left_batch: Final = batch[:midpoint]
+        right_batch: Final = batch[midpoint:]
+        left_undelivered: Final = await _keep_the_remainder_on_cancel(
+            send_batch_with_413_split(
+                batch=left_batch,
+                send_batch=send_batch,
+                exceeds_limits=exceeds_limits,
+                success_status_codes=success_status_codes,
+                integration_name=integration_name,
+                drop_error_message=drop_error_message,
+                non_success_handler=non_success_handler,
+            ),
+            right_batch,
+        )
+        if left_undelivered:
+            return (*left_undelivered, *right_batch)
+        return await send_batch_with_413_split(
+            batch=right_batch,
+            send_batch=send_batch,
+            exceeds_limits=exceeds_limits,
+            success_status_codes=success_status_codes,
+            integration_name=integration_name,
+            drop_error_message=drop_error_message,
+            non_success_handler=non_success_handler,
+        )
+
+    async def _handle_413() -> tuple[_BatchItem, ...]:
+        if len(batch) == 1:
+            verbose_logger.error(drop_error_message)
+            return ()
+        return await _halve()
+
+    if not batch:
+        return ()
+
+    try:
+        oversized: Final = exceeds_limits(batch)
+    except Exception as e:  # noqa: BLE001  # any record that cannot be serialized is isolated and dropped alone
+        if len(batch) > 1:
+            return await _halve()
+        verbose_logger.exception("%s dropped a record that cannot be serialized - %s", integration_name, e)
+        return ()
+    if oversized and len(batch) > 1:
+        return await _halve()
+
+    try:
+        response: Final = await send_batch(batch)
+    except MaskedHTTPStatusError as e:
+        if e.status_code == 413:
+            return await _handle_413()
+        return non_success_handler(batch, e.status_code, integration_name, str(e))
+    except asyncio.CancelledError as cancelled:
+        raise BatchSendCancelled(tuple(batch)) from cancelled
+    except Exception as e:
+        verbose_logger.exception("%s Error sending batch API - %s", integration_name, e)
+        return tuple(batch)
+
+    if response.status_code == 413:
+        return await _handle_413()
+    if response.status_code not in success_status_codes:
+        return non_success_handler(batch, response.status_code, integration_name, response.text)
+
+    verbose_logger.debug("%s delivered %s records, status_code=%s", integration_name, len(batch), response.status_code)
+    return ()

litellm/integrations/cloudzero/cz_stream_api.py

@@ -97,7 +97,11 @@ class CloudZeroStreamer:
                 continue
 
         # Convert lists back to DataFrames
-        return {date_key: pl.DataFrame(records) for date_key, records in daily_batches.items() if records}
+        return {
+            date_key: pl.DataFrame(records, infer_schema_length=None)
+            for date_key, records in daily_batches.items()
+            if records
+        }
 
     def _parse_and_convert_timestamp(self, timestamp_str: str) -> datetime:
         """Parse timestamp string and convert to UTC."""

litellm/integrations/cloudzero/transform.py

@@ -95,7 +95,7 @@ class CBFTransformer:
         if len(cbf_data) > 0:
             console.print(f"[green]✓ Successfully transformed {len(cbf_data):,} records[/green]")
 
-        return pl.DataFrame(cbf_data)
+        return pl.DataFrame(cbf_data, infer_schema_length=None)
 
     def _create_cbf_record(self, row: dict[str, object]) -> CBFRecord:
         """Create a single CBF record from LiteLLM daily spend row."""

litellm/integrations/custom_guardrail.py

@@ -46,6 +46,7 @@ dc: Final = DualCache()
 
 from litellm.constants import (
     GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS,
+    LOGS_GUARDRAIL_INFORMATION_MARKER,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
 )
 from litellm.exceptions import (
@@ -151,6 +152,13 @@ class CustomGuardrail(CustomLogger):
 
     records_own_guardrail_information: ClassVar[bool] = False
 
+    def __init_subclass__(cls, **kwargs: object) -> None:  # kwargs-ok: forwarded to cooperative __init_subclass__ hooks
+        super().__init_subclass__(**kwargs)
+        own_apply_guardrail: Final = cls.__dict__.get("apply_guardrail")
+        if own_apply_guardrail is None or LOGS_GUARDRAIL_INFORMATION_MARKER in vars(own_apply_guardrail):
+            return
+        cls.apply_guardrail = log_guardrail_information(own_apply_guardrail)
+
     def __init__(
         self,
         guardrail_name: str | None = None,
@@ -940,6 +948,23 @@ class CustomGuardrail(CustomLogger):
         """
         return False
 
+    def _suppressed_by_auto_router_compression(self) -> bool:
+        """True when an auto router's own compression policy suppresses this guardrail.
+
+        Reads request-scoped state set by `arm_pre_call`, never request metadata. The
+        caller controls metadata, and metadata reaches spend logs the caller can read,
+        so a suppression list carried there would be one a request could replay to
+        switch off a PII or content-filter guardrail for itself.
+        """
+        name: Final = self.guardrail_name
+        if not name:
+            return False
+        from litellm.proxy.guardrails.auto_router_compression import (
+            suppressed_compression_guardrails,
+        )
+
+        return name in suppressed_compression_guardrails()
+
     def should_run_guardrail(
         self,
         data,
@@ -948,6 +973,9 @@ class CustomGuardrail(CustomLogger):
         """
         Returns True if the guardrail should be run on the event_type
         """
+        if self._suppressed_by_auto_router_compression():
+            return False
+
         requested_guardrails: Final = self.get_guardrail_from_metadata(data)
         disable_global_guardrail: Final = self.get_disable_global_guardrail(data)
         opted_out_global_guardrails: Final = self.get_opted_out_global_guardrails_from_metadata(data)
@@ -1559,4 +1587,5 @@ def log_guardrail_information(func):
             return async_wrapper(*args, **kwargs)
         return sync_wrapper(*args, **kwargs)
 
+    vars(wrapper)[LOGS_GUARDRAIL_INFORMATION_MARKER] = True  # rebind-ok: stamps the wrapper this call just built
     return wrapper

litellm/integrations/datadog/datadog.py

@@ -29,6 +29,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.integrations.batch_utils import BatchSendCancelled, requeue_after_http_error, send_batch_with_413_split
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.datadog.datadog_handler import (
     get_datadog_base_url_from_env,
@@ -43,7 +44,6 @@ from litellm.integrations.datadog.datadog_mock_client import (
 )
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.llms.custom_httpx.http_handler import (
-    MaskedHTTPStatusError,
     _get_httpx_client,
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -396,6 +396,9 @@ class DataDogLogger(
             if self.is_mock_mode:
                 verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(batch_to_send))
 
+        except BatchSendCancelled as cancelled:
+            self.log_queue = list(cancelled.undelivered) + self.log_queue  # mutable-ok: logger queue remains appendable
+            raise asyncio.CancelledError() from cancelled
         except Exception as e:
             self.log_queue = batch_to_send + self.log_queue
             verbose_logger.exception("Datadog Error sending batch API - %s\n%s", e, traceback.format_exc())
@@ -413,53 +416,16 @@ class DataDogLogger(
         that could not be delivered because of a non-413 (transient) error, so the caller
         re-queues only those and never the events already accepted by Datadog.
         """
-        pending: Final[list[list]] = [batch]
-        while pending:
-            chunk = pending.pop()
-            if not chunk:
-                continue
-            if len(chunk) > 1 and self._exceeds_intake_limits(chunk):
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-            try:
-                response = await self.async_send_compressed_data(chunk)
-            except Exception as e:
-                if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413:
-                    response = e.response
-                else:
-                    verbose_logger.exception("Datadog Error sending batch API - %s", e)
-                    return self._undelivered(chunk, pending)
-
-            if response.status_code == 413:
-                if len(chunk) == 1:
-                    verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value)
-                    continue
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-
-            if response.status_code != 202:
-                verbose_logger.error(
-                    "Datadog: unexpected response status_code=%s, text=%s",
-                    response.status_code,
-                    response.text,
-                )
-                return self._undelivered(chunk, pending)
-
-            verbose_logger.debug(
-                "Datadog: delivered %s events, status_code=%s, text=%s",
-                len(chunk),
-                response.status_code,
-                response.text,
-            )
-        return []
-
-    @staticmethod
-    def _undelivered(chunk: list, pending: list[list]) -> list:
-        return chunk + [event for remaining in reversed(pending) for event in remaining]
+        undelivered: Final = await send_batch_with_413_split(
+            batch=batch,
+            send_batch=self.async_send_compressed_data,
+            exceeds_limits=self._exceeds_intake_limits,
+            success_status_codes=frozenset({202}),
+            integration_name="Datadog",
+            drop_error_message=DD_ERRORS.DATADOG_413_ERROR.value,
+            non_success_handler=requeue_after_http_error,
+        )
+        return list(undelivered)  # mutable-ok: caller prepends records to the logger queue
 
     @staticmethod
     def _exceeds_intake_limits(chunk: Sequence[DatadogPayload]) -> bool:
@@ -606,7 +572,7 @@ class DataDogLogger(
         )
         return dd_payload
 
-    async def async_send_compressed_data(self, data: list) -> Response:
+    async def async_send_compressed_data(self, data: Sequence[DatadogPayload]) -> Response:
         """
         Async helper to send compressed data to datadog self.intake_url
 

litellm/integrations/datadog/datadog_cost_management.py

@@ -1,6 +1,7 @@
 import asyncio
 import os
 import time
+from collections.abc import Mapping
 from datetime import datetime
 from typing import Any, Final, cast
 
@@ -181,7 +182,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
 
         # cast because StandardLoggingMetadata is a TypedDict; we iterate it
         # as a generic mapping below.
-        metadata: Final[dict[str, Any]] = cast(dict[str, Any], log.get("metadata") or {})
+        metadata: Final[Mapping[str, object]] = cast(dict[str, Any], log.get("metadata") or {})
 
         # Backwards-compat: team/user/model_group preserved regardless of allowlist.
         if metadata.get("user_api_key_alias"):
@@ -233,7 +234,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
         tags[key] = normalize_datadog_tag_value(value)
 
     @staticmethod
-    def _add_tag(tags: dict[str, str], key: str, value: Any) -> None:
+    def _add_tag(tags: dict[str, str], key: str, value: object) -> None:
         if value:
             tags[key] = str(value)
 

litellm/integrations/dotprompt/dotprompt_manager.py

@@ -4,6 +4,7 @@ Builds on top of PromptManagementBase to provide .prompt file support.
 """
 
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
@@ -347,14 +348,14 @@ class DotpromptManager(CustomPromptManagement):
         metadata: Final = json_data.get("metadata", {})
         self.prompt_manager.add_prompt(prompt_id, content, metadata)
 
-    def load_prompts_from_json(self, prompts_data: dict[str, dict[str, Any]]) -> None:
+    def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
         """Load multiple prompts from JSON data."""
         self.prompt_manager.load_prompts_from_json_data(prompts_data)
 
-    def get_prompts_as_json(self) -> dict[str, dict[str, Any]]:
+    def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
         """Get all prompts in JSON format."""
         return self.prompt_manager.get_all_prompts_as_json()
 
-    def convert_prompt_file_to_json(self, file_path: str) -> dict[str, Any]:
+    def convert_prompt_file_to_json(self, file_path: str) -> Mapping[str, object]:
         """Convert a .prompt file to JSON format."""
         return self.prompt_manager.prompt_file_to_json(file_path)

litellm/integrations/focus/destinations/s3_destination.py

@@ -3,14 +3,26 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Mapping
 from datetime import timezone
-from typing import Any, Final
+from typing import Final, TypedDict
 
 import boto3
+from typing_extensions import ReadOnly
 
 from .base import FocusDestination, FocusTimeWindow
 
 
+class _S3ClientKwargs(TypedDict, total=False):
+    """Optional boto3 client arguments the destination config may supply."""
+
+    region_name: ReadOnly[str]
+    endpoint_url: ReadOnly[str]
+    aws_access_key_id: ReadOnly[str]
+    aws_secret_access_key: ReadOnly[str]
+    aws_session_token: ReadOnly[str]
+
+
 class FocusS3Destination(FocusDestination):
     """Handles uploading serialized exports to S3 buckets."""
 
@@ -18,7 +30,7 @@ class FocusS3Destination(FocusDestination):
         self,
         *,
         prefix: str,
-        config: dict[str, Any] | None = None,
+        config: Mapping[str, str] | None = None,
     ) -> None:
         config = config or {}
         bucket_name: Final = config.get("bucket_name")
@@ -47,25 +59,23 @@ class FocusS3Destination(FocusDestination):
         key_prefix: Final = "/".join(filter(None, parts))
         return f"{key_prefix}/{filename}" if key_prefix else filename
 
+    def _client_kwargs(self) -> _S3ClientKwargs:
+        """Collect the boto3 client arguments the destination config provides."""
+        region: Final = self.config.get("region_name")
+        endpoint: Final = self.config.get("endpoint_url")
+        key_id: Final = self.config.get("aws_access_key_id")
+        secret: Final = self.config.get("aws_secret_access_key")
+        token: Final = self.config.get("aws_session_token")
+        return {
+            **(_S3ClientKwargs(region_name=region) if region else _S3ClientKwargs()),
+            **(_S3ClientKwargs(endpoint_url=endpoint) if endpoint else _S3ClientKwargs()),
+            **(_S3ClientKwargs(aws_access_key_id=key_id) if key_id else _S3ClientKwargs()),
+            **(_S3ClientKwargs(aws_secret_access_key=secret) if secret else _S3ClientKwargs()),
+            **(_S3ClientKwargs(aws_session_token=token) if token else _S3ClientKwargs()),
+        }
+
     def _upload(self, content: bytes, object_key: str) -> None:
-        client_kwargs: Final[dict[str, Any]] = {}
-        region_name: Final = self.config.get("region_name")
-        if region_name:
-            client_kwargs["region_name"] = region_name
-        endpoint_url: Final = self.config.get("endpoint_url")
-        if endpoint_url:
-            client_kwargs["endpoint_url"] = endpoint_url
-
-        session_kwargs: Final[dict[str, Any]] = {}
-        for key in (
-            "aws_access_key_id",
-            "aws_secret_access_key",
-            "aws_session_token",
-        ):
-            if self.config.get(key):
-                session_kwargs[key] = self.config[key]
-
-        s3_client: Final = boto3.client("s3", **client_kwargs, **session_kwargs)
+        s3_client: Final = boto3.client("s3", **self._client_kwargs())
         s3_client.put_object(
             Bucket=self.bucket_name,
             Key=object_key,

litellm/integrations/focus/focus_logger.py

@@ -102,7 +102,7 @@ class FocusLogger(CustomLogger):
             # No time bounds → export all available data
             await self._export_all(limit=limit)
 
-    async def dry_run_export_usage_data(self, limit: int | None = DEFAULT_DRY_RUN_LIMIT) -> dict[str, Any]:
+    async def dry_run_export_usage_data(self, limit: int | None = DEFAULT_DRY_RUN_LIMIT) -> dict[str, object]:
         """Return transformed data without uploading."""
         engine: Final = self._ensure_engine()
         return await engine.dry_run_export_usage_data(limit=limit)
@@ -153,7 +153,7 @@ class FocusLogger(CustomLogger):
             **trigger_kwargs,
         )
 
-    def _build_scheduler_trigger(self) -> dict[str, Any]:
+    def _build_scheduler_trigger(self) -> dict[str, str | int]:
         """Return scheduler configuration for the selected frequency."""
         if self.frequency == "interval":
             seconds: Final = self.interval_seconds or 60

litellm/integrations/generic_prompt_management/generic_prompt_manager.py

@@ -4,6 +4,7 @@ Fetches prompts from any API that implements the /beta/litellm_prompt_management
 """
 
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -349,7 +350,7 @@ class GenericPromptManager(CustomPromptManagement):
     def _apply_variables(
         self,
         prompt_client: PromptManagementClient,
-        variables: dict[str, Any],
+        variables: Mapping[str, object],
     ) -> PromptManagementClient:
         """
         Apply variables to the prompt template.

litellm/integrations/humanloop.py

@@ -4,7 +4,7 @@ Humanloop integration
 https://humanloop.com/
 """
 
-from typing import Any, Final, cast
+from typing import Final, cast
 
 import httpx
 from typing_extensions import TypedDict
@@ -24,7 +24,7 @@ class PromptManagementClient(TypedDict):
     prompt_id: str
     prompt_template: list[AllMessageValues]
     model: str | None
-    optional_params: dict[str, Any] | None
+    optional_params: dict[str, object] | None
 
 
 class HumanLoopPromptManager(DualCache):
@@ -36,7 +36,7 @@ class HumanLoopPromptManager(DualCache):
         return cast(PromptManagementClient | None, self.get_cache(key=humanloop_prompt_id))
 
     def _compile_prompt_helper(
-        self, prompt_template: list[AllMessageValues], prompt_variables: dict[str, Any]
+        self, prompt_template: list[AllMessageValues], prompt_variables: dict[str, object]
     ) -> list[AllMessageValues]:
         """
         Helper function to compile the prompt by substituting variables in the template.

litellm/integrations/newrelic/newrelic.py

@@ -47,6 +47,8 @@ import os
 import threading
 import time
 import uuid
+from collections.abc import Mapping, Sequence
+from datetime import datetime
 from typing import Any, Final
 
 import litellm
@@ -408,8 +410,8 @@ class NewRelicLogger(CustomLogger):
     def _get_duration(
         self,
         kwargs: dict,
-        start_time: Any,
-        end_time: Any,
+        start_time: datetime | float | None,
+        end_time: datetime | float | None,
         standard_logging_object: StandardLoggingPayload | None = None,
     ) -> float | None:
         """
@@ -438,7 +440,7 @@ class NewRelicLogger(CustomLogger):
         self,
         kwargs: dict,
         standard_logging_object: StandardLoggingPayload | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Extract request parameters like temperature and max_tokens, preferring
         StandardLoggingPayload.model_parameters.
@@ -450,7 +452,7 @@ class NewRelicLogger(CustomLogger):
         else:
             source_params = kwargs.get("optional_params") or {}
 
-        params: Final = {}
+        params: Final[dict[str, object]] = {}
 
         temperature: Final = source_params.get("temperature")
         if temperature is not None:
@@ -502,7 +504,7 @@ class NewRelicLogger(CustomLogger):
         response_model: str,
         vendor: str,
         standard_logging_object: StandardLoggingPayload | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> Sequence[Mapping[str, object]]:
         """
         Extract all messages (request + response) with sequence numbers and timestamps.
 
@@ -512,7 +514,7 @@ class NewRelicLogger(CustomLogger):
         Adds timestamps from StandardLoggingPayload (preferred) or kwargs if available
         (converted to epoch milliseconds).
         """
-        messages: Final = []
+        messages: Final[list[dict[str, object]]] = []
         sequence = 0
 
         # Extract timestamps, preferring StandardLoggingPayload
@@ -544,7 +546,7 @@ class NewRelicLogger(CustomLogger):
         else:
             request_messages = kwargs.get("messages") or []
         for msg in request_messages:
-            message_data = {
+            message_data: dict[str, object] = {
                 "role": msg.get("role") or "user",
                 "sequence": sequence,
                 "response.model": response_model,
@@ -599,11 +601,11 @@ class NewRelicLogger(CustomLogger):
         num_messages: int,
         usage: dict[str, int],
         duration: float | None = None,
-        request_params: dict[str, Any] | None = None,
+        request_params: Mapping[str, object] | None = None,
     ):
         """Record LlmChatCompletionSummary event to New Relic."""
         try:
-            event_data: Final = {
+            event_data: Final[dict[str, object]] = {
                 "id": request_id,
                 "request_id": request_id,
                 "request.model": request_model,
@@ -647,7 +649,7 @@ class NewRelicLogger(CustomLogger):
         request_id: str,
         llm_response_id: str,
         trace_id: str | None,
-        messages: list[dict[str, Any]],
+        messages: Sequence[Mapping[str, object]],
     ):
         """Record LlmChatCompletionMessage events to New Relic.
 
@@ -666,7 +668,7 @@ class NewRelicLogger(CustomLogger):
 
             for message in messages:
                 sequence = message["sequence"]
-                event_data = {
+                event_data: dict[str, object] = {
                     "id": f"{llm_response_id}-{sequence}",
                     "request_id": request_id,
                     "completion_id": request_id,

litellm/integrations/opentelemetry.py

@@ -1,7 +1,7 @@
 import os
 import threading
 from collections import OrderedDict
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
@@ -166,7 +166,7 @@ class OTELMetricAttributeFilter:
     exclude_list: list[str] | None = None
 
 
-def _build_metric_attribute_filter(value: Any) -> OTELMetricAttributeFilter:
+def _build_metric_attribute_filter(value: object) -> OTELMetricAttributeFilter:
     if isinstance(value, OTELMetricAttributeFilter):
         return value
     if not isinstance(value, dict):
@@ -205,7 +205,7 @@ def _resolve_metric_attribute_filter(
     )
 
 
-def _normalize_team_metadata_keys(value: Any) -> list[str]:
+def _normalize_team_metadata_keys(value: str | Iterable[object] | None) -> list[str]:
     """Coerce a team-metadata allowlist from a list or comma-separated string.
 
     config.yaml passes a YAML list; an env var passes a comma-separated string.
@@ -1569,7 +1569,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier)
 
     @staticmethod
-    def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None:
+    def _team_metadata_json(value: object, allowed_keys: list[str]) -> str | None:
         """JSON-serialize only the allowlisted sub-keys of a team's metadata.
 
         Returns ``None`` when nothing is allowlisted or no allowlisted key is
@@ -3524,7 +3524,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             kwargs={"standard_logging_object": {"error_information": error_information}},
         )
 
-    def set_preprocessing_duration_attribute(self, span: Span | None, container: Any) -> None:
+    def set_preprocessing_duration_attribute(self, span: Span | None, container: object) -> None:
         """
         Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
         provider handoff) on the proxy SERVER span. ``litellm_received_at``

litellm/integrations/opentelemetry_utils/gen_ai_semconv.py

@@ -117,7 +117,7 @@ class OTELGenAISemconvMixin:
     if TYPE_CHECKING:
         config: "OpenTelemetryConfig"
 
-        def safe_set_attribute(self, span: Span, key: str, value: Any) -> None: ...
+        def safe_set_attribute(self, span: Span, key: str, value: object) -> None: ...
 
         def _capture_in_event(self) -> bool: ...
 
@@ -195,13 +195,13 @@ class OTELGenAISemconvMixin:
             if value:
                 self.safe_set_attribute(span=span, key=semconv_key, value=value)
 
-    def _build_inference_details_attrs(self, kwargs: dict, response_obj: dict, provider: str) -> dict[str, Any]:
+    def _build_inference_details_attrs(self, kwargs: dict, response_obj: dict, provider: str) -> dict[str, str]:
         """Build the attribute payload for the inference-details event.
 
         Always includes provider/operation; input/output messages are added
         only when content capture is enabled and non-empty. Mixin-internal.
         """
-        attrs: Final[dict[str, Any]] = {
+        attrs: Final[dict[str, str]] = {
             "event_name": _INFERENCE_DETAILS_EVENT_NAME,
             "gen_ai.provider.name": provider,
             "gen_ai.operation.name": self._gen_ai_operation_name(kwargs),

litellm/integrations/opik/opik_payload_builder/payload_builders.py

@@ -15,8 +15,8 @@ def build_trace_payload(
     response_obj: dict[str, Any],
     start_time: datetime,
     end_time: datetime,
-    input_data: Any,
-    output_data: Any,
+    input_data: object,
+    output_data: object,
     metadata: dict[str, object],
     tags: list[str],
     thread_id: str | None,
@@ -45,8 +45,8 @@ def build_span_payload(
     response_obj: dict[str, Any],
     start_time: datetime,
     end_time: datetime,
-    input_data: Any,
-    output_data: Any,
+    input_data: object,
+    output_data: object,
     metadata: dict[str, object],
     tags: list[str],
     usage: dict[str, int],

litellm/integrations/otel/README.md

@@ -35,6 +35,15 @@ span orphaned into its own trace). The anchor — a contextvar inherited by thos
 child tasks — gives a stable parent in both cases. DB/service spans keep ambient
 parenting so an auth DB lookup still nests under `auth`.
 
+The anchor is also what `litellm.request.route` is read from: `request_root_http_route`
+returns the server span's own `http.route`, so the LLM call span cannot disagree with
+its parent about which endpoint served the request. That means the route template on a
+normal route and the literal path on a passthrough prefix, because the passthrough hook
+rewrote the attribute; an MCP call anchors the same server span, so it reports the
+`/mcp` mount point. Attributes stay readable after a span ends, so the async close
+callback reads the same value. Where no server span was anchored at all, the route the
+proxy recorded at auth (`metadata.user_api_key_request_route`) is the backstop.
+
 **Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's
 service-logging layer instruments many internal functions, but only some are
 traceable units of work:

litellm/integrations/otel/logger.py

@@ -49,6 +49,7 @@ from litellm.integrations.otel.model.utils import to_ns
 from litellm.integrations.otel.plumbing.context import (
     is_recordable_span,
     mcp_message_transport_span,
+    request_root_http_route,
     request_root_span,
     resolve_mcp_span_context,
     resolve_parent_context,
@@ -541,6 +542,7 @@ class OpenTelemetryV2(CustomLogger):
             payload,
             capture_content=self.config.capture_span_content,
             time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
+            request_route=request_root_http_route(),
         )
         end_time_ns: Final = to_ns(end_time)
         if carrier is not None and carrier.span is not None:

litellm/integrations/otel/mappers/genai.py

@@ -89,6 +89,7 @@ class GenAIMapper:
         f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
         f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
         LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
+        LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
     }
 
     _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {

litellm/integrations/otel/model/metadata.py

@@ -64,6 +64,7 @@ class RequestIdentity:
     # completes (routing has picked a deployment), so it's absent from the
     # auth-time seed and filled only from the payload.
     provider_model: str | None = None
+    request_route: str | None = None
     metadata: Mapping[str, str] = field(default_factory=dict)
 
     @classmethod
@@ -87,6 +88,7 @@ class RequestIdentity:
             key_hash=as_str(raw_meta.get("user_api_key_hash")),
             end_user=as_str(payload.get("end_user")) or as_str(raw_meta.get("user_api_key_end_user_id")),
             provider_model=resolve_provider_model(payload),
+            request_route=as_str(raw_meta.get("user_api_key_request_route")),
             metadata=metadata,
         )
 

litellm/integrations/otel/model/payloads.py

@@ -386,6 +386,7 @@ class LLMCallSpanData:
     # keeps routes the convention folds into one operation distinguishable.
     output_type: GenAIOutputType | None = None
     call_type: str | None = None
+    request_route: str | None = None
 
     @classmethod
     def from_standard_logging_payload(
@@ -393,6 +394,7 @@ class LLMCallSpanData:
         payload: StandardLoggingPayload,
         capture_content: bool = False,
         time_to_first_chunk_seconds: float | None = None,
+        request_route: str | None = None,
     ) -> LLMCallSpanData:
         params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
         # The single parse of the request's metadata — the request-vs-provider
@@ -433,6 +435,7 @@ class LLMCallSpanData:
             time_to_first_chunk_seconds=time_to_first_chunk_seconds,
             output_type=resolve_output_type(call_type),
             call_type=call_type or None,
+            request_route=request_route or context.identity.request_route,
         )
 
 

litellm/integrations/otel/model/semconv.py

@@ -295,6 +295,7 @@ class LiteLLM:
     # ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``.
     PROVIDER_MODEL: Final = "litellm.provider.model"
     REQUEST_STREAMING: Final = "litellm.request.streaming"
+    REQUEST_ROUTE: Final = "litellm.request.route"
     TOOLS_DECLARED: Final = "litellm.request.tools.declared"
     GUARDRAIL_NAME: Final = "litellm.guardrail.name"
     GUARDRAIL_MODE: Final = "litellm.guardrail.mode"

litellm/integrations/otel/plumbing/context.py

@@ -6,6 +6,7 @@ from typing import Final
 
 from opentelemetry import baggage
 from opentelemetry.context import Context, get_current
+from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.trace import (
     Link,
     NonRecordingSpan,
@@ -18,6 +19,8 @@ from opentelemetry.trace.propagation.tracecontext import (
     TraceContextTextMapPropagator,
 )
 
+from litellm.integrations.otel.model.semconv import HTTP
+
 _PROPAGATOR: Final = TraceContextTextMapPropagator()
 
 # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
@@ -55,6 +58,25 @@ def request_root_span() -> "Span | None":
     return span if is_recordable_span(span) else None
 
 
+def request_root_http_route() -> str | None:
+    """``http.route`` exactly as the request's root SERVER span reports it.
+
+    Read off the span rather than re-derived, so the LLM call span cannot disagree
+    with its own parent about which endpoint served the request: the template the
+    instrumentation matched, or the literal path where
+    ``mount._passthrough_span_name_hook`` rewrote it, are already in the attribute.
+    An MCP call anchors that same server span, so it reports the ``/mcp`` mount
+    point the instrumentation matched. Attributes stay readable after a span ends,
+    so this answers just as well from the async logging callback.
+
+    None when no server span is anchored, which is the SDK path and any deployment
+    where the FastAPI instrumentation did not mount.
+    """
+    span: Final = request_root_span()
+    route: Final = span.attributes.get(HTTP.ROUTE) if isinstance(span, ReadableSpan) and span.attributes else None
+    return route if isinstance(route, str) and route else None
+
+
 # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
 # MCP client propagated in the current request's ``params._meta``. The MCP gateway
 # sets it per message so the MCP span can record the client's span as a span

litellm/integrations/prometheus.py

@@ -10,7 +10,7 @@ import sys
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import replace
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast
 
 from pydantic import BaseModel
 
@@ -142,6 +142,9 @@ class _ExcludedLabelMetric:
         return self._metric.labels(*kept_values) if kept_values else self._metric
 
 
+_MetricLike: TypeAlias = "NoOpMetric | _ExcludedLabelMetric | MetricWrapperBase"
+
+
 def _get_budget_metrics_per_request_timeout() -> float:
     raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
     if raw is None:
@@ -1652,7 +1655,7 @@ class PrometheusLogger(CustomLogger):
 
         cache_creation_detail_tokens: Final = PrometheusLogger._resolve_cache_write_tokens(prompt_details)
 
-        detail_metrics: Final[list[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]]] = [
+        detail_metrics: Final[list[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]]] = [
             (
                 self.litellm_input_cached_tokens_metric,
                 "litellm_input_cached_tokens_metric",
@@ -1705,7 +1708,7 @@ class PrometheusLogger(CustomLogger):
         if not isinstance(usage_object, dict):
             return
 
-        media_metrics: Final[list[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]]] = [
+        media_metrics: Final[list[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]]] = [
             (
                 self.litellm_video_duration_seconds_metric,
                 "litellm_video_duration_seconds_metric",
@@ -1727,7 +1730,7 @@ class PrometheusLogger(CustomLogger):
 
     def _inc_sparse_usage_counters(
         self,
-        counters_with_values: Sequence[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]],
+        counters_with_values: Sequence[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]],
         enum_values: UserAPIKeyLabelValues,
         label_context: PrometheusLabelFactoryContext | None = None,
     ) -> None:
@@ -2607,7 +2610,7 @@ class PrometheusLogger(CustomLogger):
         for all successful requests (both streaming and non-streaming).
         """
 
-    def _safe_get(self, obj: Any, key: str, default: object = None) -> Any:
+    def _safe_get(self, obj: object, key: str, default: object = None) -> Any:
         """Get value from dict or Pydantic model."""
         if obj is None:
             return default
@@ -2623,7 +2626,7 @@ class PrometheusLogger(CustomLogger):
         """
         standard_logging_payload: Final = request_kwargs.get("standard_logging_object", {}) or {}
         _litellm_params: Final = request_kwargs.get("litellm_params", {}) or {}
-        _metadata_raw: Final = self._safe_get(standard_logging_payload, "metadata") or {}
+        _metadata_raw: Final[object] = self._safe_get(standard_logging_payload, "metadata") or {}
         if isinstance(_metadata_raw, dict):
             _metadata = _metadata_raw
         else:
@@ -4215,8 +4218,8 @@ class PrometheusLogger(CustomLogger):
 
     def _safe_duration_seconds(
         self,
-        start_time: Any,
-        end_time: Any,
+        start_time: object,
+        end_time: object,
     ) -> float | None:
         """
         Compute the duration in seconds between two objects.

litellm/integrations/shadow_eval_logger.py

@@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_judge import (
 )
 from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
 from litellm.llms.base_llm.base_utils import type_to_response_format_param
+from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
 from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
 
@@ -60,9 +61,9 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16
 _MAX_JUDGE_RESPONSE_CHARS: Final = 8_000
 _MAX_JUDGE_PROMPT_CHARS: Final = 24_000
 
-# The judge answers with a small JSON object; a tighter budget truncates the JSON
-# mid-object and the attempt is lost to an error row.
-JUDGE_MAX_OUTPUT_TOKENS: Final = 1500
+# Covers the judge's reasoning tokens as well as its small JSON answer: a judge deployment
+# carrying an elevated reasoning_effort spends a tight cap before it ever answers.
+JUDGE_MAX_OUTPUT_TOKENS: Final = 4096
 
 _MAX_ERROR_CHARS: Final = 500
 
@@ -418,6 +419,20 @@ def _failure_detail(e: BaseException) -> str:
     return f"{type(e).__name__}{location}: {e}"
 
 
+def _judge_reply_shape(response: object) -> str:
+    """How an unparseable judge reply was shaped. The parser's own message cannot separate a
+    judge that answered with nothing from one truncated mid-object, and those want opposite
+    fixes. Shape only, never the reply text: the judge quotes the sampled turns it compares,
+    and no attempt row carries sampled content today."""
+    read: Final = _chat_message_reader(response)
+    if read is None:
+        return "unreadable judge reply"
+    content: Final = read("content")
+    served: Final = str(_field_reader(response)("model") or "unknown")
+    body: Final = f"{len(str(content))} chars" if content else "no content"
+    return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}"
+
+
 def _call_cost(response: object) -> float:
     """Price one eval-arm call with the figure the spend pipeline bills: the router client
     stamps _hidden_params.response_cost from the deployment's own pricing, which the public
@@ -650,6 +665,7 @@ class ActiveShadowEvalJob(BaseModel):
     id: str
     router_name: str
     router_names: tuple[str, ...] = ()
+    models: frozenset[str] = frozenset()
     direction: ShadowEvalDirection = "forward"
     baseline_model: str | None = None
     shadow_percentage: float
@@ -692,6 +708,21 @@ class ActiveShadowEvalJob(BaseModel):
         return self.baseline_model or arm_router
 
 
+def _canonical_group(router: "Router | None", model_group: str) -> str:
+    """A model group in the one spelling both a job's scope and a request's model compare
+    under: an alias resolves to its target so the two never fail to match on spelling."""
+    return (
+        resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
+    ) or model_group
+
+
+def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:
+    """Whether the request's group is in the job's model scope. Both sides resolve through
+    the router's alias map at match time, so a re-pointed alias applies to the next request
+    rather than after the jobs cache rolls."""
+    return not job.models or any(_canonical_group(router, name) == model_group for name in job.models)
+
+
 def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
     """The sampling path's view of one job row, or None for a row it cannot sample: an
     unknown direction, or a reverse job with no baseline model to duplicate against.
@@ -714,7 +745,8 @@ class ShadowEvalLogger(CustomLogger):
     A job targets a virtual key, a team, or a user; a request qualifies for a job when
     any of its resolved identities (key hash, team id, user id) matches the job's
     target, so team and user jobs cover JWT-authenticated traffic, which carries no
-    key hash at all."""
+    key hash at all. A job scoped to model groups further requires the request's
+    requested group to be one of them."""
 
     def __init__(
         self,
@@ -801,19 +833,24 @@ class ShadowEvalLogger(CustomLogger):
         active_jobs: Sequence[ActiveShadowEvalJob],
         request_metadata: Mapping[str, object],
         request_id: str,
+        model_group: str,
     ) -> tuple[ActiveShadowEvalJob, ...]:
         """The jobs that sample this request. A key can hold one job per direction, and a
         request routed by one job's router while bypassing the other's qualifies for both;
         each is separately budgeted, so both fire. An admitting job that loses the sampling
-        dice is counted, so results can weigh judged rows against the traffic they stand for."""
+        dice is counted, so results can weigh judged rows against the traffic they stand for.
+        A request outside a job's direction or model scope is not that job's traffic and
+        goes uncounted, so the funnel stays a fraction of the traffic the job admits."""
         eligible: list[ActiveShadowEvalJob] = []  # mutable-ok: bucketed per-job admission
         now: Final = datetime.now(timezone.utc)
+        router: Final = self._router_provider()
         for job in active_jobs:
             if (
                 now >= job.ends_at
                 or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
                 or (job.max_budget is not None and job.spend >= job.max_budget)
                 or not _direction_admits(request_metadata, job)
+                or not _scope_admits(router, job, model_group)
             ):
                 continue
             if not _sample_hits(request_id, job.id, job.shadow_percentage):
@@ -868,6 +905,7 @@ class ShadowEvalLogger(CustomLogger):
                 tuple(job for target in targets for job in active_jobs.get(target, ())),
                 request_metadata,
                 request_id,
+                _canonical_group(self._router_provider(), str(payload.get("model_group") or "")),
             )
             if not eligible:
                 return
@@ -1242,7 +1280,9 @@ class ShadowEvalLogger(CustomLogger):
             verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
         except Exception as e:  # noqa: BLE001  # malformed verdicts become error rows
             verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e)
-            return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response))
+            return _CallFailure(
+                f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response)
+            )
         return _JudgeVerdict(
             preference=_unmask_preference(verdict.preference, real_is_a),
             confidence=max(0.0, min(1.0, verdict.confidence)),

litellm/integrations/weave/weave_otel.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import base64
 import json
 import os
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 from opentelemetry.trace import Status, StatusCode
@@ -59,7 +60,7 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
         safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
 
 
-def _set_weave_specific_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any):
+def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj: Any):
     """
     Sets Weave-specific metadata attributes onto the OTEL span.
 
@@ -169,7 +170,7 @@ def get_weave_otel_config() -> WeaveOtelConfig:
     )
 
 
-def set_weave_otel_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any):
+def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
     """
     Sets OpenTelemetry span attributes for Weave observability.
     Uses the same attribute setting logic as other OTEL integrations for consistency.

litellm/integrations/websearch_interception/tools.py

@@ -6,12 +6,13 @@ Native provider tools (like Anthropic's web_search_20250305) are converted
 to this format for consistent interception and execution.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
 
 
-def get_litellm_web_search_tool() -> dict[str, Any]:
+def get_litellm_web_search_tool() -> dict[str, object]:
     """
     Get the standard LiteLLM web search tool definition.
 
@@ -49,7 +50,7 @@ def get_litellm_web_search_tool() -> dict[str, Any]:
     }
 
 
-def get_litellm_web_search_tool_openai() -> dict[str, Any]:
+def get_litellm_web_search_tool_openai() -> dict[str, object]:
     """
     Get the standard LiteLLM web search tool definition in OpenAI format.
 
@@ -82,7 +83,7 @@ def get_litellm_web_search_tool_openai() -> dict[str, Any]:
     }
 
 
-def get_litellm_web_search_tool_responses() -> dict[str, Any]:
+def get_litellm_web_search_tool_responses() -> dict[str, object]:
     """
     Get the standard LiteLLM web search tool definition in Responses API format.
 
@@ -114,7 +115,7 @@ def get_litellm_web_search_tool_responses() -> dict[str, Any]:
     }
 
 
-def is_web_search_tool_responses(tool: dict[str, Any]) -> bool:
+def is_web_search_tool_responses(tool: Mapping[str, object]) -> bool:
     """
     Check if a tool is a web search tool for the Responses API.
 
@@ -195,7 +196,7 @@ def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
     return False
 
 
-def is_anthropic_native_web_search_tool(tool: dict[str, Any]) -> bool:
+def is_anthropic_native_web_search_tool(tool: Mapping[str, object]) -> bool:
     """
     Check if a tool is an Anthropic-native ``web_search_*`` tool.
 

litellm/integrations/websearch_interception/transformation.py

@@ -24,7 +24,7 @@ class WebSearchTransformation:
 
     @staticmethod
     def transform_request(
-        response: Any,
+        response: object,
         stream: bool,
         response_format: str = "anthropic",
     ) -> tuple[bool, list[dict]]:
@@ -66,7 +66,7 @@ class WebSearchTransformation:
 
     @staticmethod
     def _detect_from_responses_response(
-        response: Any,
+        response: object,
     ) -> tuple[bool, list[dict]]:
         """Parse a Responses API response for ``litellm_web_search`` function calls.
 
@@ -399,7 +399,7 @@ class WebSearchTransformation:
     def build_web_search_tool_result_block(
         tool_use_id: str,
         search_response: SearchResponse | None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Build an Anthropic-native ``web_search_tool_result`` content block.
 
@@ -433,7 +433,7 @@ class WebSearchTransformation:
                 emitted with an empty result list (signals "search ran, no
                 results" rather than "search did not run").
         """
-        items: Final[list[dict[str, Any]]] = []
+        items: Final[list[dict[str, object]]] = []
         if search_response is not None:
             results: Final = getattr(search_response, "results", None) or []
             for r in results:

litellm/interactions/agents/http_handler.py

@@ -6,7 +6,7 @@ Extends InteractionsHTTPHandler so that the shared HTTP infrastructure
 duplicated. BaseAgentsAPIConfig stays as pure transform code.
 """
 
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import Any, Final
 
 import httpx
@@ -39,11 +39,11 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
         extra_headers: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
+    ) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
         if _is_async:
             return self.async_create_agent(
                 agents_api_config=agents_api_config,
@@ -94,7 +94,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
         extra_headers: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> AgentCreateResponse:
@@ -145,7 +145,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> AgentListResponse | Coroutine[Any, Any, AgentListResponse]:
+    ) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
         if _is_async:
             return self.async_list_agents(
                 agents_api_config=agents_api_config,
@@ -220,7 +220,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
+    ) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
         if _is_async:
             return self.async_get_agent(
                 agents_api_config=agents_api_config,
@@ -299,7 +299,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> AgentDeleteResult | Coroutine[Any, Any, AgentDeleteResult]:
+    ) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
         if _is_async:
             return self.async_delete_agent(
                 agents_api_config=agents_api_config,
@@ -378,7 +378,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> AgentVersionsResponse | Coroutine[Any, Any, AgentVersionsResponse]:
+    ) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
         if _is_async:
             return self.async_list_agent_versions(
                 agents_api_config=agents_api_config,

litellm/interactions/agents/main.py

@@ -30,7 +30,7 @@ Usage:
 
 import asyncio
 import contextvars
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from functools import partial
 from typing import Any, Final
 
@@ -75,7 +75,7 @@ def _make_logging_obj(
     model: str,
     custom_llm_provider: str,
     call_type: str,
-    optional_params: dict[str, Any],
+    optional_params: dict[str, object],
 ) -> LiteLLMLoggingObj:
     litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
     litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
@@ -102,7 +102,7 @@ async def acreate(
     base_environment: InteractionEnvironment | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_body: Mapping[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
 ) -> AgentCreateResponse:
@@ -146,10 +146,10 @@ def create(
     base_environment: InteractionEnvironment | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_body: Mapping[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
-) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
+) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
     """
     Sync: Create a managed agent on the provider side.
 
@@ -244,7 +244,7 @@ def list(
     extra_headers: dict[str, Any] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
-) -> AgentListResponse | Coroutine[Any, Any, AgentListResponse]:
+) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
     """Sync: List all agents on the provider side."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@@ -320,7 +320,7 @@ def get(
     extra_headers: dict[str, Any] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
-) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
+) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
     """Sync: Get a specific agent by name."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@@ -397,7 +397,7 @@ def delete(
     extra_headers: dict[str, Any] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
-) -> AgentDeleteResult | Coroutine[Any, Any, AgentDeleteResult]:
+) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
     """Sync: Delete a specific agent by name."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@@ -474,7 +474,7 @@ def list_versions(
     extra_headers: dict[str, Any] | None = None,
     timeout: float | httpx.Timeout | None = None,
     **kwargs,
-) -> AgentVersionsResponse | Coroutine[Any, Any, AgentVersionsResponse]:
+) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
     """Sync: List versions of a specific agent."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"

litellm/interactions/http_handler.py

@@ -4,7 +4,7 @@ HTTP Handler for Interactions API requests.
 This module handles the HTTP communication for the Google Interactions API.
 """
 
-from collections.abc import AsyncIterator, Coroutine, Iterator
+from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
 from typing import Any, Final
 
 import httpx
@@ -96,8 +96,8 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         model: str | None = None,
         agent: str | None = None,
         input: InteractionInput | None = None,
-        extra_headers: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
+        extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
@@ -105,7 +105,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
     ) -> (
         InteractionsAPIResponse
         | Iterator[InteractionsAPIStreamingResponse]
-        | Coroutine[Any, Any, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
+        | Coroutine[object, object, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
     ):
         """
         Create a new interaction (synchronous or async based on _is_async flag).
@@ -211,8 +211,8 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         model: str | None = None,
         agent: str | None = None,
         input: InteractionInput | None = None,
-        extra_headers: dict[str, Any] | None = None,
-        extra_body: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
+        extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
         stream: bool | None = None,
@@ -345,11 +345,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> InteractionsAPIResponse | Coroutine[Any, Any, InteractionsAPIResponse]:
+    ) -> InteractionsAPIResponse | Coroutine[object, object, InteractionsAPIResponse]:
         """Get an interaction by ID."""
         if _is_async:
             return self.async_get_interaction(
@@ -407,7 +407,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> InteractionsAPIResponse:
@@ -464,11 +464,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> DeleteInteractionResult | Coroutine[Any, Any, DeleteInteractionResult]:
+    ) -> DeleteInteractionResult | Coroutine[object, object, DeleteInteractionResult]:
         """Delete an interaction by ID."""
         if _is_async:
             return self.async_delete_interaction(
@@ -527,7 +527,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> DeleteInteractionResult:
@@ -585,11 +585,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
-    ) -> CancelInteractionResult | Coroutine[Any, Any, CancelInteractionResult]:
+    ) -> CancelInteractionResult | Coroutine[object, object, CancelInteractionResult]:
         """Cancel an interaction by ID."""
         if _is_async:
             return self.async_cancel_interaction(
@@ -648,7 +648,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> CancelInteractionResult:

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -34,8 +34,8 @@ class LiteLLMResponsesInteractionsConfig:
         model: str,
         input: InteractionInput | None,
         optional_params: InteractionsAPIOptionalRequestParams,
-        **kwargs,
-    ) -> dict[str, Any]:
+        **kwargs: object,
+    ) -> dict[str, object]:
         """
         Transform an Interactions API request to a Responses API request.
 
@@ -45,7 +45,7 @@ class LiteLLMResponsesInteractionsConfig:
         - tools -> tools (similar format)
         - generation_config -> temperature, top_p, etc.
         """
-        responses_request: Final[dict[str, Any]] = {
+        responses_request: Final[dict[str, object]] = {
             "model": model,
         }
 
@@ -201,15 +201,15 @@ class LiteLLMResponsesInteractionsConfig:
         - Extract usage
         """
         # Extract text from outputs and build both `outputs` (legacy) and `steps` (new schema).
-        outputs: Final[list[dict[str, Any]]] = []
-        steps: Final[list[dict[str, Any]]] = []
+        outputs: Final[list[dict[str, object]]] = []
+        steps: Final[list[dict[str, object]]] = []
         if hasattr(responses_response, "output") and responses_response.output:
             for output_item in responses_response.output:
                 # Use getattr with None default to safely access content
                 content = getattr(output_item, "content", None)
                 if content is not None:
                     content_items = content if isinstance(content, list) else [content]
-                    model_output_contents: list[dict[str, Any]] = []
+                    model_output_contents: list[dict[str, object]] = []
                     for content_item in content_items:
                         # Check if content_item has text attribute
                         text = getattr(content_item, "text", None)
@@ -264,7 +264,7 @@ class LiteLLMResponsesInteractionsConfig:
         # Add usage if available
         # Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format
         # (total_input_tokens, total_output_tokens)
-        usage: Final = getattr(responses_response, "usage", None)
+        usage: Final[object] = getattr(responses_response, "usage", None)
         if usage:
             interactions_response_dict["usage"] = {
                 "total_input_tokens": getattr(usage, "input_tokens", 0),

litellm/interactions/main.py

@@ -229,7 +229,7 @@ def create(
 ) -> (
     InteractionsAPIResponse
     | Iterator[InteractionsAPIStreamingResponse]
-    | Coroutine[Any, Any, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
+    | Coroutine[object, object, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
 ):
     """
     Sync: Create a new interaction using Google's Interactions API.
@@ -406,7 +406,7 @@ def get(
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> InteractionsAPIResponse | Coroutine[Any, Any, InteractionsAPIResponse]:
+) -> InteractionsAPIResponse | Coroutine[object, object, InteractionsAPIResponse]:
     """Sync: Get an interaction by its ID."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or "gemini"
@@ -510,7 +510,7 @@ def delete(
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> DeleteInteractionResult | Coroutine[Any, Any, DeleteInteractionResult]:
+) -> DeleteInteractionResult | Coroutine[object, object, DeleteInteractionResult]:
     """Sync: Delete an interaction by its ID."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or "gemini"
@@ -612,7 +612,7 @@ def cancel(
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> CancelInteractionResult | Coroutine[Any, Any, CancelInteractionResult]:
+) -> CancelInteractionResult | Coroutine[object, object, CancelInteractionResult]:
     """Sync: Cancel an interaction by its ID."""
     local_vars: Final = locals()
     custom_llm_provider = custom_llm_provider or "gemini"

litellm/litellm_core_utils/core_helpers.py

@@ -419,7 +419,7 @@ def safe_deep_copy(data):
     if litellm.safe_memory_mode is True:
         return data
 
-    litellm_parent_otel_span: Any | None = None
+    litellm_parent_otel_span: object | None = None
     # Step 1: Remove the litellm_parent_otel_span
     litellm_parent_otel_span = None
     if isinstance(data, dict):
@@ -510,7 +510,7 @@ def independent_snapshot(
     }
 
 
-def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
+def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
     """
     Recursively filter out Exception objects and callable objects from dicts/lists.
 
@@ -542,7 +542,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
         return None
 
     if isinstance(data, dict):
-        result: Final[dict[str, Any]] = {}
+        result: Final[dict[str, object]] = {}
         for k, v in data.items():
             # Skip exception and callable values
             if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
@@ -556,7 +556,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
                 continue
         return result
     elif isinstance(data, list):
-        result_list: Final[list[Any]] = []
+        result_list: Final[list[object]] = []
         for item in data:
             # Skip exception and callable items
             if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
@@ -624,7 +624,7 @@ def redact_nested_match_and_regex_keys(
     # Iterative traversal; `seen` guards against cyclic refs preserved by deepcopy.
     try:
         seen: Final[set] = set()
-        stack: Final[list[Any]] = [redacted]
+        stack: Final[list[object]] = [redacted]
         while stack:
             node = stack.pop()
             node_id = id(node)

litellm/litellm_core_utils/dot_notation_indexing.py

@@ -23,12 +23,13 @@ Used by JWT Auth to get the user role from the token, and by
 additional_drop_params to remove nested fields from optional parameters.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final, TypeVar
 
 T = TypeVar("T")
 
 
-def get_nested_value(data: dict[str, Any], key_path: str, default: T | None = None) -> T | None:
+def get_nested_value(data: Mapping[str, object], key_path: str, default: T | None = None) -> T | None:
     """
     Retrieves a value from a nested dictionary using dot notation.
 
@@ -107,7 +108,7 @@ def _parse_path_segments(path: str) -> list:
 
 
 def _delete_nested_value_custom(
-    data: dict[str, Any] | list[Any],
+    data: dict[str, object] | list[object],
     segments: list,
     segment_index: int = 0,
 ) -> None:
@@ -168,13 +169,15 @@ def _delete_nested_value_custom(
             if segment in data:
                 next_segment: Final = segments[segment_index + 1] if segment_index + 1 < len(segments) else None
 
+                child: Final = data[segment]
+
                 # If next segment is array notation, current field should be list
                 if next_segment and (next_segment.startswith("[")):
-                    if isinstance(data[segment], list):
-                        _delete_nested_value_custom(data[segment], segments, segment_index + 1)
+                    if isinstance(child, list):
+                        _delete_nested_value_custom(child, segments, segment_index + 1)
                 # Otherwise navigate into dict
-                elif isinstance(data[segment], dict):
-                    _delete_nested_value_custom(data[segment], segments, segment_index + 1)
+                elif isinstance(child, dict):
+                    _delete_nested_value_custom(child, segments, segment_index + 1)
 
 
 def delete_nested_value(
@@ -182,7 +185,7 @@ def delete_nested_value(
     path: str,
     depth: int = 0,
     max_depth: int = 20,
-) -> dict[str, Any]:
+) -> dict[str, object]:
     """
     Delete a field from nested data using JSONPath notation.
 

litellm/litellm_core_utils/health_check_helpers.py

@@ -6,14 +6,13 @@ import base64
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Final, Literal
 
-from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
+from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, DocumentType
+from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging
     from litellm.types.utils import ImageResponse
 
-# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
-TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
 
 # Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG
 TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC"
@@ -29,6 +28,14 @@ def get_image_file_for_health_check() -> bytes:
     return base64.b64decode(TEST_IMAGE_BASE64)
 
 
+def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
+    from litellm.utils import ProviderConfigManager
+
+    provider: Final = next((known for known in LlmProviders if known.value == custom_llm_provider), None)
+    config: Final = ProviderConfigManager.get_provider_ocr_config(model=model, provider=provider) if provider else None
+    return (config or BaseOCRConfig()).get_health_check_document()
+
+
 class HealthCheckHelpers:
     @staticmethod
     async def ahealth_check_wildcard_models(
@@ -247,9 +254,6 @@ class HealthCheckHelpers:
             ),
             "ocr": lambda: litellm.aocr(
                 **_filter_model_params(model_params=model_params),
-                document={
-                    "type": "document_url",
-                    "document_url": TEST_PDF_URL,
-                },
+                document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
             ),
         }

litellm/litellm_core_utils/json_validation_rule.py

@@ -5,10 +5,10 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 
 
 def normalize_json_schema_types(
-    schema: dict[str, Any] | list[Any] | Any,
+    schema: object,
     depth: int = 0,
     max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
-) -> dict[str, Any] | list[Any] | Any:
+) -> object:
     """
     Normalize JSON schema types from uppercase to lowercase format.
 
@@ -47,7 +47,7 @@ def normalize_json_schema_types(
         return [normalize_json_schema_types(item, depth + 1, max_depth) for item in schema]
 
     if isinstance(schema, dict):
-        normalized_schema: Final[dict[str, Any]] = {}
+        normalized_schema: Final[dict[str, object]] = {}
 
         for key, value in schema.items():
             if key == "type" and isinstance(value, str) and value in type_mapping:

litellm/litellm_core_utils/litellm_logging.py

@@ -36,7 +36,7 @@ from litellm._logging import (
     verbose_logger,
 )
 from litellm._uuid import uuid
-from litellm.batches.batch_utils import _handle_completed_batch
+from litellm.batches.batch_utils import _handle_completed_batch, batch_cost_is_final
 from litellm.caching.caching import DualCache, InMemoryCache
 from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.constants import (
@@ -78,7 +78,10 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
 from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
     InteractionsUsageObjectTransformation,
 )
-from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
+from litellm.litellm_core_utils.logging_utils import (
+    truncate_base64_in_messages,
+    truncate_base64_in_messages_async,
+)
 from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
 from litellm.litellm_core_utils.redact_messages import (
     redact_message_input_output_from_custom_logger,
@@ -538,6 +541,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.standard_built_in_tools_params: StandardBuiltInToolsParams = (
             self.initialize_standard_built_in_tools_params(kwargs)
         )
+        self.truncated_messages_for_logging: str | list | dict | None = None  # mutable-ok: logged messages shape
         ## TIME TO FIRST TOKEN LOGGING ##
         self.completion_start_time: datetime.datetime | None = None
         self._llm_caching_handler: LLMCachingHandler | None = None
@@ -1820,6 +1824,7 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.aanthropic_messages.value, False) is not True
             and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
             and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
+            and litellm_params.get(CallTypes.arealtime.value, False) is not True
         )
 
     def _is_assembled_stream_success(self, result=None) -> bool:
@@ -1913,7 +1918,9 @@ class Logging(LiteLLMLoggingBaseClass):
         two paths cannot mutate it at the same time. ``prefer_async_handlers`` only
         bypasses the sync-SDK-only shortcut (e.g. ``async for`` on a stream from
         ``completion()``); legacy string callbacks still run via
-        ``executor.submit(failure_handler)`` when configured.
+        ``executor.submit(failure_handler)`` when configured, and still get submitted
+        when the awaiting task is cancelled (e.g. the event loop shuts down right after
+        the request failed).
         """
         litellm_params: Final = self.model_call_details.get("litellm_params", {}) or {}
         sync_sdk: Final = self._is_sync_litellm_request(litellm_params)
@@ -1922,12 +1929,11 @@ class Logging(LiteLLMLoggingBaseClass):
             self.failure_handler(exception, traceback_exception)
             return
 
-        await self.async_failure_handler(exception, traceback_exception)
-
-        if not self._should_run_sync_failure_callbacks_for_async_calls():
-            return
-
-        executor.submit(self.failure_handler, exception, traceback_exception)
+        try:
+            await self.async_failure_handler(exception, traceback_exception)
+        finally:
+            if self._should_run_sync_failure_callbacks_for_async_calls():
+                executor.submit(self.failure_handler, exception, traceback_exception)
 
     def should_run_logging(
         self,
@@ -2893,13 +2899,6 @@ class Logging(LiteLLMLoggingBaseClass):
             ):  # polling job will query these frequently, don't spam db logs
                 return
 
-            from litellm.proxy.openai_files_endpoints.common_utils import (
-                _is_base64_encoded_unified_file_id,
-            )
-
-            # check if file id is a unified file id
-            is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(result.id)
-
             batch_cost: Final = kwargs.get("batch_cost", None)
             batch_usage = kwargs.get("batch_usage", None)
             batch_models = kwargs.get("batch_models", None)
@@ -2907,9 +2906,7 @@ class Logging(LiteLLMLoggingBaseClass):
             batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
             has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
 
-            should_compute_batch_data: Final = (
-                not is_base64_unified_file_id or not has_explicit_batch_data and result.status == "completed"
-            )
+            should_compute_batch_data: Final = not has_explicit_batch_data and batch_cost_is_final(result)
             if has_explicit_batch_data:
                 result._hidden_params["response_cost"] = batch_cost
                 result._hidden_params["batch_models"] = batch_models
@@ -2932,6 +2929,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 result._hidden_params["batch_failed_requests"] = batch_result.failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result.usage = batch_result.usage
 
+        self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
+            StandardLoggingPayloadSetup.append_system_prompt_messages(
+                kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
+            )
+        )
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
             end_time=end_time,
@@ -3224,8 +3226,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details = {}
 
         if (
-            self.model_call_details.get("log_event_type") == "failed_api_call"
-            and self.model_call_details.get("exception") is exception
+            self.model_call_details.get("exception") is exception
             and self.model_call_details.get("standard_logging_object") is not None
         ):
             return start_time, self.model_call_details["end_time"]
@@ -3821,7 +3822,7 @@ class Logging(LiteLLMLoggingBaseClass):
     def record_streamed_anthropic_message_id(self, message_id: str) -> None:
         self.streamed_anthropic_message_id = message_id
 
-    def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
+    def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
         """
         The ModelResponse a /v1/messages spend_logs row is built from.
 
@@ -5665,8 +5666,8 @@ class StandardLoggingPayloadSetup:
             error_code=error_status,
             error_class=error_class,
             llm_provider=_llm_provider_in_exception,
-            traceback=traceback_info,
-            error_message=error_message,
+            traceback=_redact_string(traceback_info),
+            error_message=_redact_string(error_message),
             error_rate_limit_category=rate_limit_category,
             error_rate_limit_type=rate_limit_type,
             error_budget_entity_type=budget_error.entity_type if budget_error else None,
@@ -5871,6 +5872,7 @@ def _get_status_fields(
     # Mapping for legacy guardrail status values to new GuardrailStatus values
     GUARDRAIL_STATUS_MAP: Final[dict[str, GuardrailStatus]] = {
         "success": "success",
+        "guardrail_flagged": "guardrail_flagged",
         "blocked": "guardrail_intervened",  # legacy
         "guardrail_intervened": "guardrail_intervened",  # direct
         "failure": "guardrail_failed_to_respond",  # legacy
@@ -5892,6 +5894,7 @@ def _get_status_fields(
     GUARDRAIL_STATUS_SEVERITY: Final[tuple[GuardrailStatus, ...]] = (
         "not_run",
         "success",
+        "guardrail_flagged",
         "guardrail_failed_to_respond",
         "guardrail_intervened",
     )
@@ -6201,9 +6204,13 @@ def get_standard_logging_object_payload(
             model_id=_model_id,
             requester_ip_address=clean_metadata.get("requester_ip_address", None),
             user_agent=clean_metadata.get("user_agent", None),
-            messages=truncate_base64_in_messages(
-                StandardLoggingPayloadSetup.append_system_prompt_messages(
-                    kwargs=kwargs, messages=kwargs.get("messages")
+            messages=(
+                logging_obj.truncated_messages_for_logging
+                if logging_obj.truncated_messages_for_logging is not None
+                else truncate_base64_in_messages(
+                    StandardLoggingPayloadSetup.append_system_prompt_messages(
+                        kwargs=kwargs, messages=kwargs.get("messages")
+                    )
                 )
             ),
             response=final_response_obj,

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -168,7 +168,7 @@ class ResponseMetadata:
 
 
 def update_response_metadata(
-    result: Any,
+    result: object,
     logging_obj: LiteLLMLoggingObject,
     model: str | None,
     kwargs: dict,

litellm/litellm_core_utils/logging_utils.py

@@ -3,12 +3,15 @@ import functools
 import inspect
 import re
 import time
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping, Sequence
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
-from litellm.constants import MAX_BASE64_LENGTH_FOR_LOGGING
+from litellm.constants import (
+    BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
+    MAX_BASE64_LENGTH_FOR_LOGGING,
+)
 from litellm.types.utils import (
     ModelResponse,
     ModelResponseStream,
@@ -141,6 +144,39 @@ def truncate_base64_in_messages(
         return messages
 
 
+_StringTree = str | Sequence["_StringTree"] | Mapping[str, "_StringTree"] | None
+
+
+def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
+    stack: Final[list[_StringTree]] = [value]  # mutable-ok: explicit stack, recursive functions are banned in litellm/
+    while stack:
+        match stack.pop():
+            case str() as text:
+                yield text
+            case Mapping() as mapping:
+                stack.extend(mapping.values())
+            case Sequence() as items:
+                stack.extend(items)
+            case None:
+                pass
+
+
+async def truncate_base64_in_messages_async(
+    messages: str | list | dict | None,  # mutable-ok: same contract as truncate_base64_in_messages
+) -> str | list | dict | None:  # mutable-ok: same contract as truncate_base64_in_messages
+    """
+    Same result as truncate_base64_in_messages, but payloads whose string content
+    reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
+    thread so the regex pass over multi-MB base64 images does not block the event loop.
+    """
+    if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
+        return messages
+    total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
+    if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
+        return truncate_base64_in_messages(messages)
+    return await asyncio.to_thread(truncate_base64_in_messages, messages)
+
+
 # Global service logger instance to avoid recreating it
 _service_logger = None
 
@@ -184,7 +220,7 @@ def _get_parent_otel_span_from_logging_obj(
 
 
 def convert_litellm_response_object_to_str(
-    response_obj: Any | LiteLLMModelResponse,
+    response_obj: object,
 ) -> str | None:
     """
     Get the string of the response object from LiteLLM

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1708,8 +1708,8 @@ def _find_server_tool_result(
 
 def convert_to_anthropic_tool_invoke(
     tool_calls: list[ChatCompletionAssistantToolCall],
-    web_search_results: list[Any] | None = None,
-    tool_results: list[Any] | None = None,
+    web_search_results: Sequence[object] | None = None,
+    tool_results: Sequence[object] | None = None,
 ) -> list[AnthropicMessagesToolUseParam | dict[str, Any]]:
     """
     OpenAI tool invokes:
@@ -5349,7 +5349,7 @@ class NormalizedToolCall(TypedDict):
     arguments: dict[str, object]
 
 
-def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) -> dict[str, object]:
+def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) -> dict[str, object]:
     # Anthropic's tool_use blocks already carry a parsed dict in "input";
     # chat completions and the Responses API carry a JSON string that may be
     # truncated by the model, so route those through the repair-aware parser.

litellm/litellm_core_utils/realtime_errors.py

@@ -29,3 +29,11 @@ def websocket_close_reason(message: str, fallback: str) -> str:
     if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES:
         return message
     return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")
+
+
+def client_close_code(upstream_code: int) -> int:
+    from websockets.frames import EXTERNAL_CLOSE_CODES, CloseCode
+
+    if upstream_code in EXTERNAL_CLOSE_CODES or 3000 <= upstream_code < 5000:
+        return upstream_code
+    return int(CloseCode.INTERNAL_ERROR)

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,12 +1,15 @@
 import asyncio
 import json
-from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
+import traceback
+from collections.abc import Coroutine, Mapping, Sequence
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
 
 from typing_extensions import ReadOnly
 
 import litellm
-from litellm._logging import verbose_logger
+from litellm._logging import redact_internal_details_from_client_message, verbose_logger
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.types.llms.openai import (
@@ -19,9 +22,11 @@ from litellm.types.llms.openai import (
 from litellm.types.realtime import ALL_DELTA_TYPES
 
 from .litellm_logging import Logging as LiteLLMLogging
+from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
 
 if TYPE_CHECKING:
     from websockets.asyncio.client import ClientConnection
+    from websockets.exceptions import ConnectionClosed
 
     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -30,8 +35,30 @@ else:
     CLIENT_CONNECTION_CLASS = Any
 
 
-class _ClientWebSocketExceptions(Protocol):
-    ConnectionClosed: type[Exception]
+REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
+
+
+@dataclass(frozen=True, slots=True)
+class BackendClose:
+    code: int
+    reason: str
+
+    @property
+    def message(self) -> str:
+        if not self.reason:
+            return f"upstream websocket closed with code {self.code}"
+        return f"upstream websocket closed with code {self.code}: {self.reason}"
+
+
+class ClientLoopExit(Enum):
+    CLIENT_DISCONNECTED = auto()
+    BACKEND_CLOSED = auto()
+
+
+def backend_close_from(error: "ConnectionClosed") -> BackendClose:
+    if error.rcvd is None:
+        return BackendClose(code=1006, reason=str(error))
+    return BackendClose(code=error.rcvd.code, reason=error.rcvd.reason)
 
 
 class _ASGIScope(TypedDict, total=False):
@@ -69,10 +96,13 @@ class _ScopedWebSocket(Protocol):
 
 
 class _ClientWebSocket(_ScopedWebSocket, Protocol):
-    exceptions: _ClientWebSocketExceptions
-
     async def send_text(self, data: str) -> None: ...
     async def receive_text(self) -> str: ...
+    async def close(self, code: int = 1000, reason: str | None = None) -> None: ...
+
+
+class _LoggingWorker(Protocol):
+    def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: ...
 
 
 def _decode_json_object(payload: str) -> Mapping[str, object]:
@@ -108,11 +138,14 @@ class RealTimeStreaming:
         backend_uses_beta_protocol: bool | None = None,
         force_transcription_model: str | None = None,
         event_normalizer: RealtimeEventNormalizer | None = None,
+        logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
     ):
         self.websocket: _ClientWebSocket = websocket
         self.backend_ws = backend_ws
         self.logging_obj = logging_obj
+        self._logging_worker = logging_worker
         self.messages: list[OpenAIRealtimeEvents] = []
+        self._backend_sent_frames: bool = False
         self.input_message: dict = {}
         self.input_messages: list[dict[str, str]] = []
         self.session_tools: list[dict] = []
@@ -388,9 +421,10 @@ class RealTimeStreaming:
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.
-            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+            self._logging_worker.ensure_initialized_and_enqueue(
                 self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True)
             )
+            self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
 
     async def _send_to_backend(self, message: str) -> bool:
         """Send a message to the backend WebSocket.
@@ -1035,60 +1069,84 @@ class RealTimeStreaming:
             return True
         return False
 
-    async def backend_to_client_send_messages(self):
+    async def _relay_backend_messages(self) -> NoReturn:
+        while True:
+            try:
+                raw_response = await self.backend_ws.recv(decode=False)
+            except TypeError:
+                raw_response = await self.backend_ws.recv()
+            self._backend_sent_frames = True
+
+            if isinstance(raw_response, bytes):
+                try:
+                    raw_response = raw_response.decode("utf-8")
+                except UnicodeDecodeError:
+                    verbose_logger.warning("Received non-UTF-8 binary frame from backend, skipping.")
+                    continue
+
+            if self.provider_config:
+                try:
+                    await self._handle_provider_config_message(raw_response)
+                except Exception as e:
+                    verbose_logger.exception("Error processing backend message, skipping: %s", e)
+                    continue
+            else:
+                event = self._parse_backend_event(raw_response)
+                if event is None:
+                    await self.websocket.send_text(raw_response)
+                    continue
+
+                if self._should_drop_event_from_client(event):
+                    continue
+
+                if await self._handle_raw_backend_message(event, raw_response):
+                    continue
+
+                event = self._normalize_event_for_ga_client(event)
+                self.store_message(event)
+
+                if not self._client_wants_beta:
+                    await self.websocket.send_text(json.dumps(event))
+                    continue
+
+                translated = self._translate_event_to_beta(event)
+                if translated is None:
+                    continue
+                await self.websocket.send_text(json.dumps(translated))
+
+    async def backend_to_client_send_messages(self) -> BackendClose:
         import websockets
 
         try:
-            while True:
-                try:
-                    raw_response = await self.backend_ws.recv(decode=False)
-                except TypeError:
-                    raw_response = await self.backend_ws.recv()
-
-                if isinstance(raw_response, bytes):
-                    try:
-                        raw_response = raw_response.decode("utf-8")
-                    except UnicodeDecodeError:
-                        verbose_logger.warning("Received non-UTF-8 binary frame from backend, skipping.")
-                        continue
-
-                if self.provider_config:
-                    try:
-                        await self._handle_provider_config_message(raw_response)
-                    except Exception as e:
-                        verbose_logger.exception("Error processing backend message, skipping: %s", e)
-                        continue
-                else:
-                    event = self._parse_backend_event(raw_response)
-                    if event is None:
-                        await self.websocket.send_text(raw_response)
-                        continue
-
-                    if self._should_drop_event_from_client(event):
-                        continue
-
-                    if await self._handle_raw_backend_message(event, raw_response):
-                        continue
-
-                    event = self._normalize_event_for_ga_client(event)
-                    self.store_message(event)
-
-                    if not self._client_wants_beta:
-                        await self.websocket.send_text(json.dumps(event))
-                        continue
-
-                    translated = self._translate_event_to_beta(event)
-                    if translated is None:
-                        continue
-                    await self.websocket.send_text(json.dumps(translated))
-
+            await self._relay_backend_messages()
         except websockets.exceptions.ConnectionClosed as e:
             verbose_logger.exception("Connection closed in backend to client send messages - %s", e)
-        except Exception as e:
-            verbose_logger.exception("Error in backend to client send messages: %s", e)
-        finally:
+            close: Final = backend_close_from(e)
+            self._flush_unbilled_transcription_usage()
+            if self._backend_refused_session(close):
+                await self.log_backend_refusal(e)
+            else:
+                await self.log_messages()
+            return close
+        except asyncio.CancelledError:
             self._flush_unbilled_transcription_usage()
             await self.log_messages()
+            raise
+        except Exception as e:
+            verbose_logger.exception("Error in backend to client send messages: %s", e)
+            self._flush_unbilled_transcription_usage()
+            await self.log_messages()
+            return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
+
+    def _backend_refused_session(self, close: BackendClose) -> bool:
+        return close.code != 1000 and not self._backend_sent_frames
+
+    async def log_backend_refusal(self, error: Exception) -> None:
+        if not self.logging_obj:
+            return
+        self._logging_worker.ensure_initialized_and_enqueue(
+            self.logging_obj.dispatch_failure_handlers(error, traceback.format_exc(), prefer_async_handlers=True)
+        )
 
     @staticmethod
     def _detect_beta_header(websocket: _ScopedWebSocket) -> bool:
@@ -1243,11 +1301,22 @@ class RealTimeStreaming:
         item["content"] = new_content
         return item
 
-    async def client_ack_messages(self):
+    async def _receive_client_message(self) -> str | None:
+        try:
+            return await self.websocket.receive_text()
+        except Exception as e:  # noqa: BLE001  # whatever the client socket raises, the client is gone
+            verbose_logger.debug("Client disconnected: %s", e)
+            return None
+
+    async def client_ack_messages(self) -> ClientLoopExit:
+        import websockets
+
         client_event: _ClientEventFrame
         try:
             while True:
-                message = await self.websocket.receive_text()
+                message = await self._receive_client_message()
+                if message is None:
+                    return ClientLoopExit.CLIENT_DISCONNECTED
 
                 ## GUARDRAIL: intercept conversation.item.create for text-based injection.
                 guardrail_turn_detection_injected = False
@@ -1481,23 +1550,38 @@ class RealTimeStreaming:
                 if guardrail_turn_detection_injected and sent:
                     self._guardrail_turn_detection_update_sent = True
 
+        except websockets.exceptions.ConnectionClosed as e:
+            verbose_logger.debug("Backend closed while forwarding a client message: %s", e)
+            return ClientLoopExit.BACKEND_CLOSED
         except Exception as e:
             verbose_logger.debug("Error in client ack messages: %s", e)
+            return ClientLoopExit.CLIENT_DISCONNECTED
 
-    async def bidirectional_forward(self):
+    async def bidirectional_forward(self) -> None:
         forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages())
+        client_task: Final = asyncio.create_task(self.client_ack_messages())
         try:
-            await self.client_ack_messages()
-        except self.websocket.exceptions.ConnectionClosed:
-            verbose_logger.debug("Connection closed")
-            forward_task.cancel()
+            await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED)
+            if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED:
+                return
+            await self._close_client(await forward_task)
         finally:
-            if not forward_task.done():
-                forward_task.cancel()
-                try:
-                    await forward_task
-                except asyncio.CancelledError:
-                    pass
+            forward_task.cancel()
+            client_task.cancel()
+            await asyncio.gather(forward_task, client_task, return_exceptions=True)
+
+    async def _close_client(self, close: BackendClose) -> None:
+        redacted_message: Final = redact_internal_details_from_client_message(close.message)
+        redacted_reason: Final = redact_internal_details_from_client_message(close.reason)
+        try:
+            if close.code != 1000:
+                await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error"))
+            await self.websocket.close(
+                code=client_close_code(close.code),
+                reason=websocket_close_reason(redacted_reason, fallback=redacted_message),
+            )
+        except Exception as e:  # noqa: BLE001  # the client may already be gone; the session is over either way
+            verbose_logger.debug("Could not relay the upstream close to the client: %s", e)
 
 
 def client_sent_openai_beta_realtime_header(websocket: _ScopedWebSocket) -> bool:

litellm/litellm_core_utils/safe_json_dumps.py

@@ -30,7 +30,7 @@ def safe_dumps(
     def _transform(key: str | None, value: str) -> str:
         return value if value_transform is None else value_transform(key, value)
 
-    def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any:
+    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"

litellm/llms/a2a/common_utils.py

@@ -2,6 +2,7 @@
 Common utilities for A2A (Agent-to-Agent) Protocol
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 from pydantic import BaseModel
@@ -91,7 +92,7 @@ def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_d
     return " ".join(text_parts)
 
 
-def extract_text_from_a2a_response(response_dict: dict[str, Any], max_depth: int = 10) -> str:
+def extract_text_from_a2a_response(response_dict: Mapping[str, object], max_depth: int = 10) -> str:
     """
     Extract text content from A2A response result.
 

litellm/llms/anthropic/common_utils.py

@@ -1411,6 +1411,25 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages(  # mutable-ok:
     return [_flatten_web_search_results_in_message(m) for m in messages]  # mutable-ok: JSON wire format
 
 
+def _without_provider_specific_fields(block: object) -> object:
+    if not isinstance(block, dict) or "provider_specific_fields" not in block:
+        return block
+    return {k: v for k, v in block.items() if k != "provider_specific_fields"}  # mutable-ok: JSON wire format
+
+
+def _strip_provider_specific_fields_in_message(message: object) -> object:
+    if not isinstance(message, dict) or not isinstance(message.get("content"), list):
+        return message
+    content: Final = [_without_provider_specific_fields(b) for b in message["content"]]  # mutable-ok: JSON wire format
+    return {**message, "content": content}  # mutable-ok: JSON wire format
+
+
+def strip_provider_specific_fields_from_anthropic_messages(
+    messages: Sequence[object],
+) -> Sequence[object]:
+    return [_strip_provider_specific_fields_in_message(m) for m in messages]  # mutable-ok: JSON wire format
+
+
 def _normalized_cache_control(cache_control: object) -> dict[str, str] | None:  # mutable-ok: JSON wire format
     if not isinstance(cache_control, Mapping):
         return None

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -4,13 +4,14 @@ import copy
 import json
 import traceback
 from collections import deque
-from collections.abc import AsyncIterator, Iterator, Sequence
+from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
 from typing import (
     TYPE_CHECKING,
     Any,
     Final,
     Literal,
     Protocol,
+    cast,  # noqa: TID251  # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union
     get_args,
 )
 
@@ -100,6 +101,10 @@ class _CombinedChunkSplitter:
     @staticmethod
     def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         choices: Final = _optional_attr_sequence(chunk, "choices")
         if not choices:
             return False
@@ -114,6 +119,7 @@ class _CombinedChunkSplitter:
             or _optional_attr(delta, "tool_calls")
             or _optional_attr(delta, "reasoning_content")
             or _optional_attr(delta, "thinking_blocks")
+            or openai_chat_refusal_text(delta)
         )
 
     _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@@ -305,6 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         # Synthesized compaction block from compact_20260112 polyfill (streaming).
         self.compaction_block = compaction_block
         self.iterations_usage = iterations_usage
+        self._refusal_text: str = ""
         self.sent_compaction_block: bool = False
         # Per-phase flags so the compaction block's start/delta/stop events
         # are emitted (and the public state machine is advanced) in
@@ -423,7 +430,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         augmented["usage"] = augmented_usage
         return augmented
 
-    def _next_compaction_event(self) -> dict[str, Any] | None:
+    def _next_compaction_event(self) -> dict[str, object] | None:
         """Return the next compaction content-block SSE event, or ``None``.
 
         Anthropic delivers compaction as a single delta (no token-by-token
@@ -462,7 +469,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                 "delta": {"type": "compaction_delta", "content": summary_content},
             }
 
-        stop_event: Final = {
+        stop_event: Final[dict[str, object]] = {
             "type": "content_block_stop",
             "index": compaction_index,
         }
@@ -572,6 +579,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -806,6 +814,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -993,8 +1002,33 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     def _increment_content_block_index(self):
         self.current_content_block_index += 1
 
+    def _with_refusal_stop_details(
+        self,
+        processed_chunk: ContentBlockDelta | MessageBlockDelta,
+    ) -> ContentBlockDelta | MessageBlockDelta:
+        if processed_chunk.get("type") != "message_delta" or not self._refusal_text:
+            return processed_chunk
+        delta: Final = cast(Mapping[str, object], processed_chunk["delta"])  # cast-ok: keys checked before use
+        if delta.get("stop_reason") == "max_tokens":
+            return processed_chunk
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            refusal_stop_details,
+        )
+
+        return cast(  # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch
+            ContentBlockDelta | MessageBlockDelta,
+            {  # mutable-ok: fresh translation payload; never mutated after construction
+                **processed_chunk,
+                "delta": {  # mutable-ok: fresh message_delta payload; never mutated after construction
+                    **delta,
+                    "stop_reason": "refusal",
+                    "stop_details": refusal_stop_details(self._refusal_text),
+                },
+            },
+        )
+
     @staticmethod
-    def _delta_has_content(processed_chunk: dict[str, Any]) -> bool:
+    def _delta_has_content(processed_chunk: Mapping[str, object]) -> bool:
         """Return True if a translated chunk carries a non-empty
         ``content_block_delta`` payload.
 
@@ -1035,6 +1069,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
         from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1044,6 +1081,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "content", None):
             return False
+        if openai_chat_refusal_text(delta):
+            return False
         if getattr(delta, "reasoning_content", None):
             return False
         # thinking_blocks whose entries are all empty AND unsigned must not
@@ -1067,13 +1106,19 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         - Different content types in the response
         - Specific markers in the content
         """
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
-        # Example logic - customize based on your needs:
-        # If chunk indicates a tool call
         if chunk.choices[0].finish_reason is not None:
             return False
 
+        refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
+        if refusal_text is not None:
+            self._refusal_text = self._refusal_text + refusal_text
+
         (
             block_type,
             content_block_start,