Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit_7048_batch_cost_row_once

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,38 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
+        return 0
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/down.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0

.github/e2e-stack/secrets_to_env.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
+
+
+def main() -> int:
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
+        return 1
+    for value in secrets.values():
+        if len(value) >= MIN_MASKED_LENGTH:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -0,0 +1,44 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/workflows/test-code-quality.yml

@@ -74,6 +74,9 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-e2e-changed.yml

@@ -0,0 +1,240 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
+          tests="$(printf '%s\n' "${files}" \
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no changed e2e test files supported by this stack; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          umask 077
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        id: boot
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
+        run: |
+          umask 077
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
+            status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
+            set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
+            echo "pass ${pass} of 3 passed"
+          done
+
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi

Dockerfile

@@ -67,6 +67,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -89,6 +90,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

docker/Dockerfile.database

@@ -65,6 +65,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -87,6 +88,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

docker/Dockerfile.non_root

@@ -71,6 +71,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
+    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -99,6 +100,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -109,6 +111,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13; \
     fi
 

gateway/Dockerfile

@@ -47,6 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -59,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
+        --extra mongodb \
         --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

litellm-proxy-extras/litellm_proxy_extras/migrations/20260905120000_add_per_server_oauth_discovery_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "per_server_oauth_discovery" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -343,6 +343,7 @@ model LiteLLM_MCPServerTable {
   delegate_auth_to_upstream Boolean @default(false)
   oauth_passthrough Boolean @default(false)
   dcr_bridge Boolean?
+  per_server_oauth_discovery Boolean @default(false)
   is_byok               Boolean  @default(false)
   byok_description      String[] @default([])
   byok_api_key_help_url String?

litellm/__init__.py

@@ -13,7 +13,6 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
 ### INIT VARIABLES #########################
 import threading
 import os
-import sys
 
 # Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
 import dotenv as _dotenv
@@ -46,6 +45,8 @@ from typing import (
     TYPE_CHECKING,
     Union,
 )
+from litellm.types.integrations.datadog import DatadogInitParams
+from litellm.types.integrations.newrelic import NewRelicInitParams
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,
@@ -94,7 +95,8 @@ from litellm.constants import (
     DEFAULT_SOFT_BUDGET,
     DEFAULT_ALLOWED_FAILS,
 )
-# httpx is lazy-loaded via __getattr__
+import httpx
+
 # register_async_client_cleanup is lazy-loaded and called on first access
 
 litellm_mode = os.getenv("LITELLM_MODE", "DEV")  # "PRODUCTION", "DEV"
@@ -362,6 +364,8 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {}
 include_cost_in_streaming_usage: bool = False
 reasoning_auto_summary: bool = False
 ### PROMPTS ####
+from litellm.types.prompts.init_prompts import PromptSpec
+
 prompt_name_config_map: Dict[str, PromptSpec] = {}
 
 ##################
@@ -1267,203 +1271,206 @@ openai_video_generation_models = ["sora-2"]
 # get_llm_provider is lazy-loaded via __getattr__
 # remove_index_from_tool_calls is lazy-loaded via __getattr__
 
-# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__
-# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING
-# so static type checkers still see them
-if TYPE_CHECKING:
-    _key_management_settings: KeyManagementSettings
+# Import KeyManagementSettings here (before utils import) because _key_management_settings
+# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
+from litellm.types.secret_managers.main import KeyManagementSettings
 
-    from .utils import client
+_key_management_settings: KeyManagementSettings = KeyManagementSettings()
 
-    from .llms.custom_llm import CustomLLM
-    from .llms.anthropic.common_utils import AnthropicModelInfo
-    from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
-    from .llms.deprecated_providers.palm import (
-        PalmConfig,
-    )  # here to prevent breaking changes
-    from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
-    from .llms.gemini.common_utils import GeminiModelInfo
+# client must be imported immediately as it's used as a decorator at function definition time
+from .utils import client
 
-    from .llms.vertex_ai.vertex_embeddings.transformation import (
-        VertexAITextEmbeddingConfig,
-    )
+# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
+# (which imports tiktoken) at import time
 
-    vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
+from .llms.custom_llm import CustomLLM
+from .llms.anthropic.common_utils import AnthropicModelInfo
+from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
+from .llms.deprecated_providers.palm import (
+    PalmConfig,
+)  # here to prevent breaking changes
+from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
+from .llms.gemini.common_utils import GeminiModelInfo
 
-    from .llms.bedrock.embed.amazon_titan_v2_transformation import (
-        AmazonTitanV2Config,
-    )
-    from .llms.topaz.common_utils import TopazModelInfo
 
-    # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
-    # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
-    from .llms.xai.common_utils import XAIModelInfo
+from .llms.vertex_ai.vertex_embeddings.transformation import (
+    VertexAITextEmbeddingConfig,
+)
 
-    # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
-    # All remaining configs are now lazy loaded - see _lazy_imports_registry.py
+vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
 
-    # Import LlmProviders here (before main import) because it's imported during import time
-    # in multiple places including openai.py (via main import)
 
-    ## Lazy loading this is not straightforward, will leave it here for now.
-    from .main import *
-    from .compression import compress
+from .llms.bedrock.embed.amazon_titan_v2_transformation import (
+    AmazonTitanV2Config,
+)
+from .llms.topaz.common_utils import TopazModelInfo
 
-    # Skills API
-    from .skills.main import (
-        create_skill,
-        acreate_skill,
-        list_skills,
-        alist_skills,
-        get_skill,
-        aget_skill,
-        delete_skill,
-        adelete_skill,
-    )
-    from .evals.main import (
-        create_eval,
-        acreate_eval,
-        list_evals,
-        alist_evals,
-        get_eval,
-        aget_eval,
-        delete_eval,
-        adelete_eval,
-        cancel_eval,
-        acancel_eval,
-        create_run,
-        acreate_run,
-        list_runs,
-        alist_runs,
-        get_run,
-        aget_run,
-        delete_run,
-        adelete_run,
-        cancel_run,
-        acancel_run,
-    )
-    from .integrations import *
-    from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
-    from .exceptions import (
-        AuthenticationError,
-        InvalidRequestError,
-        BadRequestError,
-        ImageFetchError,
-        NotFoundError,
-        PermissionDeniedError,
-        RateLimitError,
-        RateLimitErrorCategory,
-        RateLimitType,
-        ServiceUnavailableError,
-        BadGatewayError,
-        OpenAIError,
-        ContextWindowExceededError,
-        ContentPolicyViolationError,
-        BudgetExceededError,
-        APIError,
-        Timeout,
-        APIConnectionError,
-        UnsupportedParamsError,
-        APIResponseValidationError,
-        UnprocessableEntityError,
-        InternalServerError,
-        JSONSchemaValidationError,
-        LITELLM_EXCEPTION_TYPES,
-        MockException,
-    )
-    from .budget_manager import BudgetManager
-    from .proxy.proxy_cli import run_server
-    from .router import Router
-    from .assistants.main import *
-    from .batches.main import *
-    from .images.main import *
-    from .videos.main import *
-    from .batch_completion.main import *
-    from .rerank_api.main import *
-    from .llms.anthropic.experimental_pass_through.messages.handler import *
-    from .responses.main import *
+# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
+# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
+from .llms.xai.common_utils import XAIModelInfo
 
-    # Interactions API is available as litellm.interactions module
-    # Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
-    from . import interactions
-    from .interactions.agents.main import (
-        acreate as acreate_agent,
-        create as create_agent,
-        alist as alist_agents,
-        list as list_agents,
-        aget as aget_agent,
-        get as get_agent,
-        adelete as adelete_agent,
-        delete as delete_agent,
-        alist_versions as alist_agent_versions,
-        list_versions as list_agent_versions,
-    )
-    from .skills.main import (
-        create_skill,
-        acreate_skill,
-        list_skills,
-        alist_skills,
-        get_skill,
-        aget_skill,
-        delete_skill,
-        adelete_skill,
-    )
-    from .containers.main import *
-    from .ocr.main import *
-    from .rust_bridge import rust
-    from .rag.main import *
-    from .sandbox.main import *
-    from .search.main import *
-    from .realtime_api.main import (
-        _arealtime,
-        acreate_realtime_client_secret,
-        acreate_realtime_transcription_session,
-        arealtime_calls,
-    )
-    from .responses.main import _aresponses_websocket
-    from .fine_tuning.main import *
-    from .files.main import *
-    from .vector_store_files.main import (
-        acreate as avector_store_file_create,
-        adelete as avector_store_file_delete,
-        alist as avector_store_file_list,
-        aretrieve as avector_store_file_retrieve,
-        aretrieve_content as avector_store_file_content,
-        aupdate as avector_store_file_update,
-        create as vector_store_file_create,
-        delete as vector_store_file_delete,
-        list as vector_store_file_list,
-        retrieve as vector_store_file_retrieve,
-        retrieve_content as vector_store_file_content,
-        update as vector_store_file_update,
-    )
-    from .scheduler import *
+# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
+# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
 
-    ### ADAPTERS ###
-    import litellm.anthropic_interface as anthropic
+# Import LlmProviders here (before main import) because it's imported during import time
+# in multiple places including openai.py (via main import)
+from litellm.types.utils import LlmProviders
 
-    ### Vector Store Registry ###
+## Lazy loading this is not straightforward, will leave it here for now.
+from .main import *
+from .compression import compress
 
-    ### RAG ###
-    from . import rag
+# Skills API
+from .skills.main import (
+    create_skill,
+    acreate_skill,
+    list_skills,
+    alist_skills,
+    get_skill,
+    aget_skill,
+    delete_skill,
+    adelete_skill,
+)
+from .evals.main import (
+    create_eval,
+    acreate_eval,
+    list_evals,
+    alist_evals,
+    get_eval,
+    aget_eval,
+    delete_eval,
+    adelete_eval,
+    cancel_eval,
+    acancel_eval,
+    create_run,
+    acreate_run,
+    list_runs,
+    alist_runs,
+    get_run,
+    aget_run,
+    delete_run,
+    adelete_run,
+    cancel_run,
+    acancel_run,
+)
+from .integrations import *
+from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
+from .exceptions import (
+    AuthenticationError,
+    InvalidRequestError,
+    BadRequestError,
+    ImageFetchError,
+    NotFoundError,
+    PermissionDeniedError,
+    RateLimitError,
+    RateLimitErrorCategory,
+    RateLimitType,
+    ServiceUnavailableError,
+    BadGatewayError,
+    OpenAIError,
+    ContextWindowExceededError,
+    ContentPolicyViolationError,
+    BudgetExceededError,
+    APIError,
+    Timeout,
+    APIConnectionError,
+    UnsupportedParamsError,
+    APIResponseValidationError,
+    UnprocessableEntityError,
+    InternalServerError,
+    JSONSchemaValidationError,
+    LITELLM_EXCEPTION_TYPES,
+    MockException,
+)
+from .budget_manager import BudgetManager
+from .proxy.proxy_cli import run_server
+from .router import Router
+from .assistants.main import *
+from .batches.main import *
+from .images.main import *
+from .videos.main import *
+from .batch_completion.main import *
+from .rerank_api.main import *
+from .llms.anthropic.experimental_pass_through.messages.handler import *
+from .responses.main import *
 
-    ### CUSTOM LLMs ###
-
-    ### CLI UTILITIES ###
-    from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
-
-    ### PASSTHROUGH ###
-    from .passthrough import allm_passthrough_route, llm_passthrough_route
-    from .google_genai import agenerate_content
+# Interactions API is available as litellm.interactions module
+# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
+from . import interactions
+from .interactions.agents.main import (
+    acreate as acreate_agent,
+    create as create_agent,
+    alist as alist_agents,
+    list as list_agents,
+    aget as aget_agent,
+    get as get_agent,
+    adelete as adelete_agent,
+    delete as delete_agent,
+    alist_versions as alist_agent_versions,
+    list_versions as list_agent_versions,
+)
+from .skills.main import (
+    create_skill,
+    acreate_skill,
+    list_skills,
+    alist_skills,
+    get_skill,
+    aget_skill,
+    delete_skill,
+    adelete_skill,
+)
+from .containers.main import *
+from .ocr.main import *
+from .rust_bridge import rust
+from .rag.main import *
+from .sandbox.main import *
+from .search.main import *
+from .realtime_api.main import (
+    _arealtime,
+    acreate_realtime_client_secret,
+    acreate_realtime_transcription_session,
+    arealtime_calls,
+)
+from .responses.main import _aresponses_websocket
+from .fine_tuning.main import *
+from .files.main import *
+from .vector_store_files.main import (
+    acreate as avector_store_file_create,
+    adelete as avector_store_file_delete,
+    alist as avector_store_file_list,
+    aretrieve as avector_store_file_retrieve,
+    aretrieve_content as avector_store_file_content,
+    aupdate as avector_store_file_update,
+    create as vector_store_file_create,
+    delete as vector_store_file_delete,
+    list as vector_store_file_list,
+    retrieve as vector_store_file_retrieve,
+    retrieve_content as vector_store_file_content,
+    update as vector_store_file_update,
+)
+from .scheduler import *
 
 ### ADAPTERS ###
+from .types.adapter import AdapterItem
+import litellm.anthropic_interface as anthropic
+
 adapters: List[AdapterItem] = []
 
 ### Vector Store Registry ###
+from .vector_stores.vector_store_registry import (
+    VectorStoreRegistry,
+    VectorStoreIndexRegistry,
+)
+
 vector_store_registry: Optional[VectorStoreRegistry] = None
 vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
 
+### RAG ###
+from . import rag
+
 ### CUSTOM LLMs ###
+from .types.llms.custom_llm import CustomLLMItem
+
 custom_provider_map: List[CustomLLMItem] = []
 _custom_providers: List[str] = []  # internal helper util, used to track names of custom providers
 disable_hf_tokenizer_download: Optional[bool] = (
@@ -1471,6 +1478,13 @@ disable_hf_tokenizer_download: Optional[bool] = (
 )
 global_disable_no_log_param: bool = False
 
+### CLI UTILITIES ###
+from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
+
+### PASSTHROUGH ###
+from .passthrough import allm_passthrough_route, llm_passthrough_route
+from .google_genai import agenerate_content
+
 ### GLOBAL CONFIG ###
 global_bitbucket_config: Optional[Dict[str, Any]] = None
 
@@ -1494,21 +1508,10 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
-    import httpx
-
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
     from litellm.caching.caching import Cache
-    from litellm.types.adapter import AdapterItem
-    from litellm.types.integrations.datadog import DatadogInitParams
-    from litellm.types.integrations.newrelic import NewRelicInitParams
-    from litellm.types.llms.custom_llm import CustomLLMItem
-    from litellm.types.prompts.init_prompts import PromptSpec
-    from litellm.vector_stores.vector_store_registry import (
-        VectorStoreIndexRegistry,
-        VectorStoreRegistry,
-    )
 
     # Type stubs for lazy-loaded configs to help mypy
     from .llms.bedrock.chat.converse_transformation import (
@@ -2184,6 +2187,16 @@ if TYPE_CHECKING:
 # Track if async client cleanup has been registered (for lazy loading)
 _async_client_cleanup_registered = False
 
+# Eager loading for backwards compatibility with VCR and other HTTP recording tools
+# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
+# For now, this only affects encoding (tiktoken) as it was the only reported issue
+# See: https://github.com/BerriAI/litellm/issues/18659
+# This ensures encoding is initialized before VCR starts recording HTTP requests
+if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
+    # Load encoding at import time (pre-#18070 behavior)
+    # This ensures encoding is initialized before VCR starts recording
+    from .main import encoding
+
 
 def __getattr__(name: str) -> Any:
     """Lazy import handler with cached registry for improved performance."""
@@ -2263,8 +2276,6 @@ def __getattr__(name: str) -> Any:
         "openAIGPT5Config": "OpenAIGPT5Config",
         "nvidiaNimConfig": "NvidiaNimConfig",
         "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
-        "vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig",
-        "_key_management_settings": "KeyManagementSettings",
     }
     if name in _config_instances:
         from ._lazy_imports import get_litellm_globals
@@ -2382,30 +2393,7 @@ def __getattr__(name: str) -> Any:
 
         return locals()[name]
 
-    from ._lazy_imports import lazy_import_litellm_submodule
-
-    submodule: Final = lazy_import_litellm_submodule(name)
-    if submodule is not None:
-        return submodule
-
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
-from ._lazy_imports import LiteLLMModule
-from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES
-
-sys.modules[__name__].__class__ = LiteLLMModule
-
-__all__ = list(STAR_IMPORT_PUBLIC_NAMES)  # mutable-ok: star imports require __all__ to be a list of str
-
-
 # ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
-
-# Eager loading for backwards compatibility with VCR and other HTTP recording tools
-# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
-# For now, this only affects encoding (tiktoken) as it was the only reported issue
-# See: https://github.com/BerriAI/litellm/issues/18659
-# This ensures encoding is initialized before VCR starts recording HTTP requests
-# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import
-if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
-    from .main import encoding

litellm/_lazy_imports.py

@@ -16,10 +16,9 @@ until they're actually needed.
 """
 
 import importlib
-import importlib.util
 import sys
 from collections.abc import Callable, Mapping
-from types import MappingProxyType, ModuleType
+from types import ModuleType
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from typing_extensions import ReadOnly, TypedDict
@@ -35,8 +34,6 @@ from ._lazy_imports_registry import (
     _LITELLM_LOGGING_IMPORT_MAP,
     _LLM_CONFIGS_IMPORT_MAP,
     _LLM_PROVIDER_LOGIC_IMPORT_MAP,
-    _SDK_MODULE_ALIASES,
-    _SDK_SYMBOLS_IMPORT_MAP,
     _TOKEN_COUNTER_IMPORT_MAP,
     _TYPES_IMPORT_MAP,
     _TYPES_UTILS_IMPORT_MAP,
@@ -81,10 +78,7 @@ def _get_utils_globals() -> dict[str, object]:
     This is where we cache imported attributes so we don't import them twice.
     When you do `litellm.utils.some_function`, it gets stored in this dictionary.
     """
-    cached: Final = sys.modules.get("litellm.utils")
-    if cached is not None:
-        return cached.__dict__
-    return importlib.import_module("litellm.utils").__dict__
+    return sys.modules["litellm.utils"].__dict__
 
 
 def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
@@ -220,10 +214,6 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
             _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
         for name in UTILS_MODULE_NAMES:
             _LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
-        for name in _SDK_SYMBOLS_IMPORT_MAP:
-            _LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols)
-        for name in _SDK_MODULE_ALIASES:
-            _LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias)
 
     return _LAZY_IMPORT_REGISTRY
 
@@ -360,86 +350,6 @@ def _lazy_import_llm_provider_logic(name: str) -> object:
     return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
 
 
-def _lazy_import_sdk_symbols(name: str) -> object:
-    """Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py"""
-    return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols")
-
-
-def _lazy_import_sdk_module_alias(name: str) -> object:
-    """Handler for litellm attributes that bind a module (e.g. litellm.anthropic)"""
-    _globals: Final = get_litellm_globals()
-    if name in _globals:
-        return _globals[name]
-    module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name])
-    _globals[name] = module  # rebind-ok: caches the resolved module alias on the package
-    return module
-
-
-_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType(
-    {
-        "batch_completion": ("litellm.batch_completion.main", "batch_completion"),
-        "ocr": ("litellm.ocr.main", "ocr"),
-        "responses": ("litellm.responses.main", "responses"),
-        "search": ("litellm.search.main", "search"),
-    }
-)
-
-
-def _shadowable_function_property(name: str) -> property:
-    """Property keeping litellm.<name> bound to the SDK function even after the import
-    machinery binds the identically named litellm.<name> subpackage onto the litellm module."""
-    module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name]
-
-    def _get(module: ModuleType) -> object:
-        stored: Final = module.__dict__.get(name)
-        if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"):
-            return stored
-        value: Final = _module_attribute(importlib.import_module(module_path), attr_name)
-        module.__dict__[name] = value  # rebind-ok: caches the resolved function on the litellm module
-        return value
-
-    def _set(module: ModuleType, value: object) -> None:
-        module.__dict__[name] = value  # rebind-ok: property setter must store assignments on the module
-
-    return property(_get, _set)
-
-
-class LiteLLMModule(ModuleType):
-    """Module type installed on the litellm package so function names shadowed by
-    same-named subpackages (litellm.responses, ...) keep resolving to the functions."""
-
-    batch_completion = _shadowable_function_property("batch_completion")
-    ocr = _shadowable_function_property("ocr")
-    responses = _shadowable_function_property("responses")
-    search = _shadowable_function_property("search")
-
-
-def lazy_import_submodule(package: str, name: str) -> "ModuleType | None":
-    """Resolve <package>.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
-    if name.startswith("__") or not name.isidentifier():
-        return None
-    qualified_name: Final = f"{package}.{name}"
-    try:
-        spec: Final = importlib.util.find_spec(qualified_name)
-    except ModuleNotFoundError:
-        return None
-    if spec is None:
-        return None
-    try:
-        module: Final = importlib.import_module(qualified_name)
-    except ModuleNotFoundError as exc:
-        if exc.name == qualified_name:
-            return None
-        raise
-    sys.modules[package].__dict__[name] = module  # rebind-ok: caches the resolved submodule on the package
-    return module
-
-
-def lazy_import_litellm_submodule(name: str) -> "ModuleType | None":
-    """Resolve litellm.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
-    return lazy_import_submodule("litellm", name)
-
-
 def _lazy_import_utils_module(name: str) -> object:
     """
     Handler for utils module lazy imports.

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -16,23 +16,32 @@ import asyncio
 import os
 import time
 import traceback
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeVar
 from urllib.parse import urlparse
 
 from litellm._logging import verbose_logger
+from litellm.integrations.batch_utils import (
+    BatchSendCancelled,
+    send_batch_with_413_split,
+    undelivered_after_http_error,
+)
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
+    MaskedHTTPStatusError,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.types.integrations.azure_sentinel import AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
 from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 DEFAULT_AZURE_AUTHORITY_HOST: Final = "https://login.microsoftonline.com"
 DEFAULT_AZURE_MONITOR_SCOPE: Final = "https://monitor.azure.com/.default"
 
+_QueuedPayload = TypeVar("_QueuedPayload", StandardLoggingPayload, StandardAuditLogPayload)
+
 MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
     {
         "login.microsoftonline.com": DEFAULT_AZURE_MONITOR_SCOPE,
@@ -153,6 +162,8 @@ class AzureSentinelLogger(CustomBatchLogger):
         asyncio.create_task(self.periodic_flush())
         self.log_queue: list[StandardLoggingPayload] = []
         self.audit_log_queue: list[StandardAuditLogPayload] = []
+        self.logs_awaiting_retry = False
+        self.audit_logs_awaiting_retry = False
 
     @staticmethod
     def _normalize_authority_host(authority_host: str) -> str:
@@ -245,8 +256,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -275,8 +286,8 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.log_queue.append(standard_logging_payload)
 
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            if len(self.log_queue) >= self.batch_size and not self.logs_awaiting_retry:
+                await self._threshold_send_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc())
@@ -298,12 +309,24 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             self.audit_log_queue.append(audit_log)
 
-            if len(self.audit_log_queue) >= self.batch_size:
-                await self.async_send_audit_batch()
+            if len(self.audit_log_queue) >= self.batch_size and not self.audit_logs_awaiting_retry:
+                await self._threshold_send_audit_logs()
 
         except Exception as e:
             verbose_logger.exception("Azure Sentinel Audit Log Layer Error - %s\n%s", e, traceback.format_exc())
 
+    async def _threshold_send_logs(self) -> None:
+        async with self.flush_lock:
+            if self.logs_awaiting_retry or len(self.log_queue) < self.batch_size:
+                return
+            await self.async_send_batch()
+
+    async def _threshold_send_audit_logs(self) -> None:
+        async with self.flush_lock:
+            if self.audit_logs_awaiting_retry or len(self.audit_log_queue) < self.batch_size:
+                return
+            await self.async_send_audit_batch()
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -311,67 +334,110 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.log_queue,
-            api_endpoint=self.api_endpoint,
-            log_type="logs",
-        )
+        batch_to_send: Final = tuple(self.log_queue)
+        self.log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.api_endpoint,
+                log_type="logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.log_queue = self._requeue(cancelled.undelivered, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.log_queue = self._requeue(batch_to_send, self.log_queue, "logs")
+            self.logs_awaiting_retry = bool(self.log_queue)
+            raise
+        self.log_queue = self._requeue(undelivered, self.log_queue, "logs")
+        self.logs_awaiting_retry = bool(undelivered) and bool(self.log_queue)
 
     async def async_send_audit_batch(self):
         """
         Sends the batch of audit logs to Azure Monitor Logs Ingestion API
         """
-        await self._async_send_batch_to_api(
-            log_queue=self.audit_log_queue,
-            api_endpoint=self.audit_api_endpoint,
-            log_type="audit logs",
+        batch_to_send: Final = tuple(self.audit_log_queue)
+        self.audit_log_queue = []  # mutable-ok: queue ownership is detached before the async send
+        try:
+            undelivered: Final = await self._async_send_batch_to_api(
+                log_queue=batch_to_send,
+                api_endpoint=self.audit_api_endpoint,
+                log_type="audit logs",
+            )
+        except BatchSendCancelled as cancelled:
+            self.audit_log_queue = self._requeue(cancelled.undelivered, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise asyncio.CancelledError() from cancelled
+        except asyncio.CancelledError:
+            self.audit_log_queue = self._requeue(batch_to_send, self.audit_log_queue, "audit logs")
+            self.audit_logs_awaiting_retry = bool(self.audit_log_queue)
+            raise
+        self.audit_log_queue = self._requeue(undelivered, self.audit_log_queue, "audit logs")
+        self.audit_logs_awaiting_retry = bool(undelivered) and bool(self.audit_log_queue)
+
+    def _requeue(
+        self,
+        undelivered: tuple[_QueuedPayload, ...],
+        queue: list[_QueuedPayload],
+        log_type: str,
+    ) -> list[_QueuedPayload]:
+        merged: Final = [*undelivered, *queue]  # mutable-ok: queue trimming returns a mutable logger queue
+        overflow: Final = len(merged) - self.max_queue_size
+        if overflow <= 0:
+            return merged
+
+        verbose_logger.warning(
+            "Azure Sentinel: %s queue exceeded max_queue_size=%s, dropped %s oldest records",
+            log_type,
+            self.max_queue_size,
+            overflow,
         )
+        return merged[overflow:]
 
     async def _async_send_batch_to_api(
         self,
-        log_queue: list[StandardLoggingPayload | StandardAuditLogPayload],
+        log_queue: tuple[_QueuedPayload, ...],
         api_endpoint: str,
         log_type: str,
-    ) -> None:
+    ) -> tuple[_QueuedPayload, ...]:
+        if not log_queue:
+            return ()
+
+        verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
         try:
-            if not log_queue:
-                return
-
-            verbose_logger.debug("Azure Sentinel - about to flush %s %s", len(log_queue), log_type)
-
-            # Get OAuth2 token
             bearer_token: Final = await self._get_oauth_token()
+        except MaskedHTTPStatusError as e:
+            return undelivered_after_http_error(log_queue, e.status_code, "Azure Sentinel OAuth token", str(e))
+        except Exception as e:
+            verbose_logger.exception("Azure Sentinel Error getting OAuth token - %s", e)
+            return tuple(log_queue)
 
-            # Convert log queue to JSON array format expected by Logs Ingestion API
-            # Each log entry should be a JSON object in the array
-            body: Final = safe_dumps(log_queue)
+        headers: Final = {
+            "Authorization": f"Bearer {bearer_token}",
+            "Content-Type": "application/json",
+        }
 
-            # Set headers for Logs Ingestion API
-            headers: Final = {
-                "Authorization": f"Bearer {bearer_token}",
-                "Content-Type": "application/json",
-            }
-
-            # Send the request
-            response = await self.async_httpx_client.post(url=api_endpoint, data=body.encode("utf-8"), headers=headers)
-
-            if response.status_code not in [200, 204]:
-                verbose_logger.error(
-                    "Azure Sentinel API error: status_code=%s, response=%s",
-                    response.status_code,
-                    response.text,
-                )
-                raise Exception(f"Failed to send logs to Azure Sentinel: {response.status_code} - {response.text}")
-
-            verbose_logger.debug(
-                "Azure Sentinel: Response from API status_code: %s",
-                response.status_code,
+        async def _send_batch(batch: Sequence[_QueuedPayload]):
+            body: Final = safe_dumps(batch)
+            return await self.async_httpx_client.post(
+                url=api_endpoint,
+                data=body.encode("utf-8"),
+                headers=headers,
             )
 
-        except Exception as e:
-            verbose_logger.exception("Azure Sentinel Error sending batch API - %s\n%s", e, traceback.format_exc())
-        finally:
-            log_queue.clear()
+        return await send_batch_with_413_split(
+            batch=log_queue,
+            send_batch=_send_batch,
+            exceeds_limits=lambda batch: (
+                len(batch) > self.batch_size
+                or len(safe_dumps(batch).encode("utf-8")) > AZURE_SENTINEL_MAX_PAYLOAD_SIZE_BYTES
+            ),
+            success_status_codes=frozenset({200, 204}),
+            integration_name="Azure Sentinel",
+            drop_error_message="Azure Sentinel API Error - Payload too large for a single record",
+            non_success_handler=undelivered_after_http_error,
+        )
 
     async def flush_queue(self):
         if self.flush_lock is None:

litellm/integrations/batch_utils.py

@@ -0,0 +1,160 @@
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from typing import Final, Generic, TypeVar
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
+
+_BatchItem = TypeVar("_BatchItem")
+
+_RETRYABLE_CLIENT_STATUS_CODES: Final = frozenset({408, 429})
+
+
+def is_retryable_status(status_code: int) -> bool:
+    return not 400 <= status_code < 500 or status_code in _RETRYABLE_CLIENT_STATUS_CODES
+
+
+def undelivered_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    """The records to requeue after a non-2xx: all of them on a status a retry can clear, none on
+    a 4xx that would only repeat, since retaining those retries a misconfiguration forever."""
+    if is_retryable_status(status_code):
+        verbose_logger.error(
+            "%s API error: status_code=%s, will retry %s records - %s",
+            integration_name,
+            status_code,
+            len(batch),
+            detail,
+        )
+        return tuple(batch)
+    verbose_logger.error(
+        "%s API error: status_code=%s is not retryable, dropped %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return ()
+
+
+def requeue_after_http_error(
+    batch: Sequence[_BatchItem],
+    status_code: int,
+    integration_name: str,
+    detail: str,
+) -> tuple[_BatchItem, ...]:
+    verbose_logger.error(
+        "%s API error: status_code=%s, will retry %s records - %s",
+        integration_name,
+        status_code,
+        len(batch),
+        detail,
+    )
+    return tuple(batch)
+
+
+class BatchSendCancelled(asyncio.CancelledError, Generic[_BatchItem]):
+    """Cancellation of a batch send, carrying only the records the destination never accepted.
+
+    A batch split under the size cap is delivered in pieces, so requeueing all of it after a
+    cancellation partway through would send the accepted pieces a second time.
+    """
+
+    def __init__(self, undelivered: tuple[_BatchItem, ...]) -> None:
+        super().__init__()
+        self.undelivered: Final = undelivered
+
+
+async def _keep_the_remainder_on_cancel(
+    send: Awaitable[tuple[_BatchItem, ...]],
+    remainder: Sequence[_BatchItem],
+) -> tuple[_BatchItem, ...]:
+    try:
+        return await send
+    except BatchSendCancelled as cancelled:
+        raise BatchSendCancelled((*cancelled.undelivered, *remainder)) from cancelled
+
+
+async def send_batch_with_413_split(
+    batch: Sequence[_BatchItem],
+    send_batch: Callable[[Sequence[_BatchItem]], Awaitable[httpx.Response]],
+    exceeds_limits: Callable[[Sequence[_BatchItem]], bool],
+    success_status_codes: frozenset[int],
+    integration_name: str,
+    drop_error_message: str,
+    non_success_handler: Callable[
+        [Sequence[_BatchItem], int, str, str], tuple[_BatchItem, ...]
+    ] = requeue_after_http_error,
+) -> tuple[_BatchItem, ...]:
+    async def _halve() -> tuple[_BatchItem, ...]:
+        midpoint: Final = len(batch) // 2
+        left_batch: Final = batch[:midpoint]
+        right_batch: Final = batch[midpoint:]
+        left_undelivered: Final = await _keep_the_remainder_on_cancel(
+            send_batch_with_413_split(
+                batch=left_batch,
+                send_batch=send_batch,
+                exceeds_limits=exceeds_limits,
+                success_status_codes=success_status_codes,
+                integration_name=integration_name,
+                drop_error_message=drop_error_message,
+                non_success_handler=non_success_handler,
+            ),
+            right_batch,
+        )
+        if left_undelivered:
+            return (*left_undelivered, *right_batch)
+        return await send_batch_with_413_split(
+            batch=right_batch,
+            send_batch=send_batch,
+            exceeds_limits=exceeds_limits,
+            success_status_codes=success_status_codes,
+            integration_name=integration_name,
+            drop_error_message=drop_error_message,
+            non_success_handler=non_success_handler,
+        )
+
+    async def _handle_413() -> tuple[_BatchItem, ...]:
+        if len(batch) == 1:
+            verbose_logger.error(drop_error_message)
+            return ()
+        return await _halve()
+
+    if not batch:
+        return ()
+
+    try:
+        oversized: Final = exceeds_limits(batch)
+    except Exception as e:  # noqa: BLE001  # any record that cannot be serialized is isolated and dropped alone
+        if len(batch) > 1:
+            return await _halve()
+        verbose_logger.exception("%s dropped a record that cannot be serialized - %s", integration_name, e)
+        return ()
+    if oversized and len(batch) > 1:
+        return await _halve()
+
+    try:
+        response: Final = await send_batch(batch)
+    except MaskedHTTPStatusError as e:
+        if e.status_code == 413:
+            return await _handle_413()
+        return non_success_handler(batch, e.status_code, integration_name, str(e))
+    except asyncio.CancelledError as cancelled:
+        raise BatchSendCancelled(tuple(batch)) from cancelled
+    except Exception as e:
+        verbose_logger.exception("%s Error sending batch API - %s", integration_name, e)
+        return tuple(batch)
+
+    if response.status_code == 413:
+        return await _handle_413()
+    if response.status_code not in success_status_codes:
+        return non_success_handler(batch, response.status_code, integration_name, response.text)
+
+    verbose_logger.debug("%s delivered %s records, status_code=%s", integration_name, len(batch), response.status_code)
+    return ()

litellm/integrations/datadog/datadog.py

@@ -29,6 +29,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.integrations.batch_utils import BatchSendCancelled, requeue_after_http_error, send_batch_with_413_split
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.datadog.datadog_handler import (
     get_datadog_base_url_from_env,
@@ -43,7 +44,6 @@ from litellm.integrations.datadog.datadog_mock_client import (
 )
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.llms.custom_httpx.http_handler import (
-    MaskedHTTPStatusError,
     _get_httpx_client,
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -396,6 +396,9 @@ class DataDogLogger(
             if self.is_mock_mode:
                 verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(batch_to_send))
 
+        except BatchSendCancelled as cancelled:
+            self.log_queue = list(cancelled.undelivered) + self.log_queue  # mutable-ok: logger queue remains appendable
+            raise asyncio.CancelledError() from cancelled
         except Exception as e:
             self.log_queue = batch_to_send + self.log_queue
             verbose_logger.exception("Datadog Error sending batch API - %s\n%s", e, traceback.format_exc())
@@ -413,53 +416,16 @@ class DataDogLogger(
         that could not be delivered because of a non-413 (transient) error, so the caller
         re-queues only those and never the events already accepted by Datadog.
         """
-        pending: Final[list[list]] = [batch]
-        while pending:
-            chunk = pending.pop()
-            if not chunk:
-                continue
-            if len(chunk) > 1 and self._exceeds_intake_limits(chunk):
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-            try:
-                response = await self.async_send_compressed_data(chunk)
-            except Exception as e:
-                if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413:
-                    response = e.response
-                else:
-                    verbose_logger.exception("Datadog Error sending batch API - %s", e)
-                    return self._undelivered(chunk, pending)
-
-            if response.status_code == 413:
-                if len(chunk) == 1:
-                    verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value)
-                    continue
-                mid = len(chunk) // 2
-                pending.append(chunk[mid:])
-                pending.append(chunk[:mid])
-                continue
-
-            if response.status_code != 202:
-                verbose_logger.error(
-                    "Datadog: unexpected response status_code=%s, text=%s",
-                    response.status_code,
-                    response.text,
-                )
-                return self._undelivered(chunk, pending)
-
-            verbose_logger.debug(
-                "Datadog: delivered %s events, status_code=%s, text=%s",
-                len(chunk),
-                response.status_code,
-                response.text,
-            )
-        return []
-
-    @staticmethod
-    def _undelivered(chunk: list, pending: list[list]) -> list:
-        return chunk + [event for remaining in reversed(pending) for event in remaining]
+        undelivered: Final = await send_batch_with_413_split(
+            batch=batch,
+            send_batch=self.async_send_compressed_data,
+            exceeds_limits=self._exceeds_intake_limits,
+            success_status_codes=frozenset({202}),
+            integration_name="Datadog",
+            drop_error_message=DD_ERRORS.DATADOG_413_ERROR.value,
+            non_success_handler=requeue_after_http_error,
+        )
+        return list(undelivered)  # mutable-ok: caller prepends records to the logger queue
 
     @staticmethod
     def _exceeds_intake_limits(chunk: Sequence[DatadogPayload]) -> bool:
@@ -606,7 +572,7 @@ class DataDogLogger(
         )
         return dd_payload
 
-    async def async_send_compressed_data(self, data: list) -> Response:
+    async def async_send_compressed_data(self, data: Sequence[DatadogPayload]) -> Response:
         """
         Async helper to send compressed data to datadog self.intake_url
 

litellm/litellm_core_utils/litellm_logging.py

@@ -5666,8 +5666,8 @@ class StandardLoggingPayloadSetup:
             error_code=error_status,
             error_class=error_class,
             llm_provider=_llm_provider_in_exception,
-            traceback=traceback_info,
-            error_message=error_message,
+            traceback=_redact_string(traceback_info),
+            error_message=_redact_string(error_message),
             error_rate_limit_category=rate_limit_category,
             error_rate_limit_type=rate_limit_type,
             error_budget_entity_type=budget_error.entity_type if budget_error else None,

litellm/llms/anthropic/common_utils.py

@@ -1411,6 +1411,25 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages(  # mutable-ok:
     return [_flatten_web_search_results_in_message(m) for m in messages]  # mutable-ok: JSON wire format
 
 
+def _without_provider_specific_fields(block: object) -> object:
+    if not isinstance(block, dict) or "provider_specific_fields" not in block:
+        return block
+    return {k: v for k, v in block.items() if k != "provider_specific_fields"}  # mutable-ok: JSON wire format
+
+
+def _strip_provider_specific_fields_in_message(message: object) -> object:
+    if not isinstance(message, dict) or not isinstance(message.get("content"), list):
+        return message
+    content: Final = [_without_provider_specific_fields(b) for b in message["content"]]  # mutable-ok: JSON wire format
+    return {**message, "content": content}  # mutable-ok: JSON wire format
+
+
+def strip_provider_specific_fields_from_anthropic_messages(
+    messages: Sequence[object],
+) -> Sequence[object]:
+    return [_strip_provider_specific_fields_in_message(m) for m in messages]  # mutable-ok: JSON wire format
+
+
 def _normalized_cache_control(cache_control: object) -> dict[str, str] | None:  # mutable-ok: JSON wire format
     if not isinstance(cache_control, Mapping):
         return None

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -1346,7 +1346,7 @@ class LiteLLMAnthropicMessagesAdapter:
                     # Add provider_specific_fields if signature is present
                     if provider_specific_fields:
                         tool_use_block.provider_specific_fields = provider_specific_fields
-                    new_content.append(tool_use_block.model_dump())
+                    new_content.append(tool_use_block.model_dump(exclude_none=True))
 
         return new_content
 

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -18,6 +18,7 @@ from litellm.llms.anthropic.common_utils import (
     flatten_unencrypted_web_search_results_in_anthropic_messages,
     sanitize_tool_use_ids_in_anthropic_messages,
     strip_empty_content_blocks_from_anthropic_messages,
+    strip_provider_specific_fields_from_anthropic_messages,
 )
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -650,7 +651,7 @@ def anthropic_messages_handler(
 
     return base_llm_http_handler.anthropic_messages_handler(
         model=model,
-        messages=messages,
+        messages=strip_provider_specific_fields_from_anthropic_messages(messages),
         anthropic_messages_provider_config=anthropic_messages_provider_config,
         anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params),
         _is_async=is_async,

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -647,7 +647,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         id=item.call_id or item.id or "",
                         name=item.name,
                         input=input_data,
-                    ).model_dump()
+                    ).model_dump(exclude_none=True)
                 )
                 stop_reason = "tool_use"
 
@@ -676,7 +676,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             id=item.get("call_id") or item.get("id", ""),
                             name=item.get("name", ""),
                             input=input_data,
-                        ).model_dump()
+                        ).model_dump(exclude_none=True)
                     )
                     stop_reason = "tool_use"
 

litellm/model_prices_and_context_window_backup.json

@@ -12238,6 +12238,48 @@
         "output_cost_per_token": 2.65e-06,
         "supports_pdf_input": true
     },
+    "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": {
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 8172,
+        "max_tokens": 8172,
+        "mode": "embedding",
+        "input_cost_per_token": 1.62e-07,
+        "input_cost_per_image": 7.2e-05,
+        "input_cost_per_video_per_second": 0.00084,
+        "input_cost_per_audio_per_second": 0.000168,
+        "output_cost_per_token": 0.0,
+        "output_vector_size": 3072,
+        "supports_embedding_image_input": true,
+        "supports_image_input": true,
+        "supports_video_input": true,
+        "supports_audio_input": true
+    },
+    "bedrock/us-gov-west-1/amazon.nova-lite-v1:0": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 300000,
+        "max_output_tokens": 10000,
+        "max_tokens": 10000,
+        "mode": "chat",
+        "output_cost_per_token": 2.88e-07,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 10000,
+        "max_tokens": 10000,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-07,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true
+    },
     "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": {
         "input_cost_per_token": 9.6e-07,
         "litellm_provider": "bedrock",
@@ -43648,6 +43690,23 @@
         "input_cost_per_token_batches": 1.65e-06,
         "output_cost_per_token_batches": 8.25e-06
     },
+    "us-gov.anthropic.claude-3-haiku-20240307-v1:0": {
+        "deprecation_date": "2026-09-10",
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "cache_read_input_token_cost": 3e-08,
+        "cache_creation_input_token_cost": 3.75e-07
+    },
     "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 4.5e-06,
         "cache_creation_input_token_cost_above_1hr": 7.2e-06,
@@ -43742,6 +43801,160 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "us-gov.anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "us-gov.anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
+    "us-gov.nvidia.nemotron-nano-3-30b": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.88e-07,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_function_calling": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.nvidia.nemotron-nano-12b-v2": {
+        "input_cost_per_token": 2.4e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 7.2e-07,
+        "supports_system_messages": true,
+        "supports_vision": true
+    },
+    "us-gov.nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
+    "us-gov.nvidia.nemotron-super-3-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 7.8e-07,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.openai.gpt-oss-20b-1:0": {
+        "input_cost_per_token": 8.4e-08,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.openai.gpt-oss-120b-1:0": {
+        "input_cost_per_token": 1.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 7.2e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "us-gov.xai.grok-4.6": {
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "au.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.375e-06,
         "cache_creation_input_token_cost_above_1hr": 2.2e-06,
@@ -59547,6 +59760,16 @@
         "supports_system_messages": true,
         "supports_vision": true
     },
+    "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
     "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": {
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "bedrock",
@@ -59651,6 +59874,70 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "bedrock/us-gov-west-1/anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
     "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": {
         "input_cost_per_token": 7.2e-08,
         "litellm_provider": "bedrock",
@@ -59676,6 +59963,16 @@
         "supports_system_messages": true,
         "supports_vision": true
     },
+    "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": {
+        "input_cost_per_token": 7.2e-08,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 2.76e-07,
+        "supports_system_messages": true
+    },
     "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": {
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "bedrock",
@@ -59780,6 +60077,70 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
     },
+    "bedrock/us-gov-east-1/anthropic.claude-opus-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.2e-05,
+        "cache_read_input_token_cost": 6e-07,
+        "input_cost_per_token": 6e-06,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": {
+        "cache_creation_input_token_cost": 1.5e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-05,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 1.2e-05,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-05,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_forced_tool_use": false,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 512
+    },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": {
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
@@ -59894,6 +60255,120 @@
         "output_cost_per_token": 3e-06,
         "cache_read_input_token_cost": 2.4e-07
     },
+    "bedrock_mantle/us-gov-west-1/xai.grok-4.6": {
+        "use_openai_responses_path": true,
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": {
+        "input_cost_per_token": 4.8e-08,
+        "output_cost_per_token": 9.6e-08,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": {
+        "input_cost_per_token": 1.56e-07,
+        "output_cost_per_token": 4.8e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/google.gemma-4-31b": {
+        "input_cost_per_token": 1.68e-07,
+        "output_cost_per_token": 4.8e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
+        "input_cost_per_token": 8.4e-08,
+        "output_cost_per_token": 3.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "output_cost_per_token": 7.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": {
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
@@ -59921,6 +60396,63 @@
         "cache_read_input_token_cost": 3.3e-07,
         "output_cost_per_token": 1.98e-05
     },
+    "bedrock_mantle/us-gov-east-1/xai.grok-4.6": {
+        "use_openai_responses_path": true,
+        "input_cost_per_token": 2.64e-06,
+        "output_cost_per_token": 7.92e-06,
+        "cache_read_input_token_cost": 6.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
+        "input_cost_per_token": 8.4e-08,
+        "output_cost_per_token": 3.6e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
+        "input_cost_per_token": 1.8e-07,
+        "output_cost_per_token": 7.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "azure/us-gov/gpt-5.1": {
         "cache_read_input_token_cost": 1.71875e-07,
         "default_reasoning_effort": "none",

litellm/models/mcp_server.py

@@ -98,6 +98,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
     delegate_auth_to_upstream: bool = False
     oauth_passthrough: bool = False
     dcr_bridge: bool | None = None
+    per_server_oauth_discovery: bool = False
     is_byok: bool = False
     byok_description: list[str] = Field(default_factory=list)
     byok_api_key_help_url: str | None = None

litellm/proxy/__init__.py

@@ -1,11 +1 @@
-from types import ModuleType
-from typing import Final
-
-
-def __getattr__(name: str) -> ModuleType:
-    from litellm._lazy_imports import lazy_import_submodule
-
-    submodule: Final = lazy_import_submodule(__name__, name)
-    if submodule is not None:
-        return submodule
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+from . import *

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -197,7 +197,7 @@ def _gateway_dcr_challenge_target(
     if targets is None:
         return None
     server: Final = global_mcp_server_manager.get_mcp_server_by_name(targets[0], client_ip=client_ip)
-    if server is None or not server.is_gateway_managed_oauth2:
+    if server is None or not server.advertises_gateway_authorization_server:
         return None
     return targets[0]
 

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -1481,11 +1481,19 @@ async def _persist_dcr_client_registration(
         )
         updated_row: Final = await update_mcp_server(
             prisma_client=prisma_client,
-            data=UpdateMCPServerRequest(
-                server_id=mcp_server.server_id,
-                credentials=credentials,
-                oauth2_flow="authorization_code",
-                **({"token_url": mcp_server.token_url} if mcp_server.token_url else {}),
+            data=(
+                UpdateMCPServerRequest(
+                    server_id=mcp_server.server_id,
+                    credentials=credentials,
+                    oauth2_flow="authorization_code",
+                    token_url=mcp_server.token_url,
+                )
+                if mcp_server.token_url
+                else UpdateMCPServerRequest(
+                    server_id=mcp_server.server_id,
+                    credentials=credentials,
+                    oauth2_flow="authorization_code",
+                )
             ),
             touched_by="mcp_oauth_dcr",
         )
@@ -2367,7 +2375,7 @@ async def _build_oauth_protected_resource_response(
     if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange:
         _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource")
 
-    if explicitly_named and mcp_server is not None and mcp_server.is_gateway_managed_oauth2:
+    if explicitly_named and mcp_server is not None and mcp_server.advertises_gateway_authorization_server:
         return {
             "authorization_servers": [f"{request_base_url}/mcp"],
             "resource": resource_url,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -155,6 +155,7 @@ from litellm.proxy._types import (
     MCPTransportType,
     SpecialMCPServerNames,
     UserAPIKeyAuth,
+    is_per_server_oauth_discovery_eligible,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
@@ -344,6 +345,7 @@ class MCPServerConfig(TypedDict, total=False):
     token_endpoint_auth_method: MCPTokenEndpointAuthMethod
     scopes: str | Sequence[str]
     dcr_bridge: object
+    per_server_oauth_discovery: ReadOnly[object]
     extra_headers: _StringList
     allowed_tools: _StringList
     disallowed_tools: _StringList
@@ -414,6 +416,31 @@ def _blank_to_none(value: str | None) -> str | None:
     return value.strip() or None
 
 
+def _config_per_server_oauth_discovery(
+    server_config: MCPServerConfig,
+    server_ref: str,
+    auth_type: MCPAuthType | None,
+    oauth2_flow: object,
+) -> bool:
+    match server_config.get("per_server_oauth_discovery", False):
+        case bool() as enabled:
+            pass
+        case other:
+            raise ValueError(
+                f"Invalid config for MCP server '{server_ref}': per_server_oauth_discovery must be a boolean "
+                f"(got {other!r})."
+            )
+    relay_eligible: Final = is_per_server_oauth_discovery_eligible(
+        auth_type, oauth2_flow, server_config.get("delegate_auth_to_upstream", False)
+    )
+    if enabled and not relay_eligible:
+        raise ValueError(
+            f"Invalid config for MCP server '{server_ref}': per_server_oauth_discovery is only supported for "
+            "auth_type oauth2 with oauth2_flow authorization_code and without delegate_auth_to_upstream."
+        )
+    return enabled
+
+
 def _pinned_config_server_id(raw_server_id: object, server_name: str) -> str | None:
     """Return the ``server_id`` an admin pinned for this config.yaml server, or ``None`` when absent.
 
@@ -2307,6 +2334,9 @@ class MCPServerManager:
                 )
 
             config_dcr_bridge = server_config.get("dcr_bridge", None)
+            config_per_server_oauth_discovery = _config_per_server_oauth_discovery(
+                server_config, server_name or server_id, auth_type, config_oauth2_flow
+            )
             if config_dcr_bridge is not None and not isinstance(config_dcr_bridge, bool):
                 raise ValueError(
                     f"Invalid config for MCP server '{server_name or server_id}': dcr_bridge "
@@ -2378,6 +2408,7 @@ class MCPServerManager:
                 delegate_auth_to_upstream=bool(server_config.get("delegate_auth_to_upstream", False)),
                 oauth_passthrough=bool(server_config.get("oauth_passthrough", False)),
                 dcr_bridge=config_dcr_bridge,
+                per_server_oauth_discovery=config_per_server_oauth_discovery,
                 # AWS SigV4 fields
                 aws_access_key_id=server_config.get("aws_access_key_id", None),
                 aws_secret_access_key=server_config.get("aws_secret_access_key", None),
@@ -2903,6 +2934,7 @@ class MCPServerManager:
             delegate_auth_to_upstream=bool(getattr(mcp_server, "delegate_auth_to_upstream", False)),
             oauth_passthrough=bool(getattr(mcp_server, "oauth_passthrough", False)),
             dcr_bridge=getattr(mcp_server, "dcr_bridge", None),
+            per_server_oauth_discovery=bool(getattr(mcp_server, "per_server_oauth_discovery", False)),
             created_at=getattr(mcp_server, "created_at", None),
             updated_at=getattr(mcp_server, "updated_at", None),
             tool_name_to_display_name=_deserialize_json_dict(getattr(mcp_server, "tool_name_to_display_name", None)),
@@ -6692,6 +6724,7 @@ class MCPServerManager:
             registration_url=server.configured_registration_url or server.registration_url,
             oauth2_flow=server.oauth2_flow,
             dcr_bridge=server.dcr_bridge,
+            per_server_oauth_discovery=server.per_server_oauth_discovery,
             token_exchange_endpoint=server.token_exchange_endpoint,
             audience=server.audience,
             subject_token_type=server.subject_token_type,
@@ -6810,6 +6843,7 @@ class MCPServerManager:
             delegate_auth_to_upstream=server.delegate_auth_to_upstream,
             oauth_passthrough=getattr(server, "oauth_passthrough", False),
             dcr_bridge=server.dcr_bridge,
+            per_server_oauth_discovery=server.per_server_oauth_discovery,
             is_byok=server.is_byok,
             byok_description=server.byok_description,
             byok_api_key_help_url=server.byok_api_key_help_url,

litellm/proxy/_experimental/mcp_server/utils.py

@@ -752,7 +752,7 @@ def interpolate_headers(headers: Mapping[str, str], variables: Mapping[str, str]
 def build_env_var_setup_url(server_id: str) -> str:
     """The frontend URL where a user can fill in their per-user env vars."""
     base: Final = os.environ.get("PROXY_BASE_URL", "").rstrip("/")
-    path: Final = f"/ui/?page=mcp-servers&fill_env_vars={quote(server_id, safe='')}"
+    path: Final = f"/ui/mcp-servers?fill_env_vars={quote(server_id, safe='')}"
     return f"{base}{path}" if base else path
 
 

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+2:I[92825,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
+3:I[216370,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js"],"default"]
+4:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
+2:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
+3:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+5:I[27201,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
+0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,11 +1,11 @@
 1:"$Sreact.fragment"
-2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
-3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
-4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
+2:I[363178,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
+3:I[12985,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
+4:I[867271,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[557951,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
+6:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+7:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+8:I[713354,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
+0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"912qRXFjlEYHK3EAPXXTc"}