Merge pull request #50280 from BerriAI/litellm_backport_translation_p0_lts_1_108_x

fix: backport #46870, #46984 and #47100 to lts/1.108.x

.circleci/scripts/run_integration.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [ "${GITHUB_ACTIONS:-}" = true ]; then
-  echo "Integration contracts are owned by CircleCI" >&2
-  exit 1
-fi
-
 suite="${1:?integration suite required}"
 mode="${2:-standard}"
 side="${3:-}"
+requested_workers="${INTEGRATION_WORKERS:-}"
+license_env=()
+if [ -n "${LITELLM_LICENSE:-}" ]; then
+  license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
+fi
 if [ "$mode" = replica ]; then
   results="test-results/integration-${suite}-replica"
 elif [ "$mode" = parity ]; then
@@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
   export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
   test -f "$LITELLM_UI_PATH/index.html"
 fi
-export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
+export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
 export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
 
 uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@@ -107,7 +107,7 @@ sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
 sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
-for service in postgres-db redis-cache; do
+for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
   address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
@@ -145,6 +145,9 @@ fi
 if [ "$suite" = mcp ]; then
   export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
 fi
+if [ -n "$requested_workers" ]; then
+  export INTEGRATION_WORKERS="$requested_workers"
+fi
 isolated_workers=false
 if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
   isolated_workers=true
@@ -185,7 +188,7 @@ start_proxy() {
     "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
-    LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+    "${license_env[@]}" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
     "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   "${shared_services[@]}" "${worker_coverage[@]}" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   LITELLM_MODE=PRODUCTION \
-  LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+  "${license_env[@]}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

.github/scripts/assert_secret_absent.py

@@ -0,0 +1,119 @@
+import base64
+import io
+import os
+import sys
+import urllib.parse
+import zipfile
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from functools import reduce
+from itertools import chain
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, TextIO
+
+SECRET_VARIABLE: Final = "SCANNED_SECRET"
+MIN_ENCODED_LENGTH: Final = 12
+MAX_ARCHIVE_DEPTH: Final = 3
+MASK: Final = "***"
+
+Forms = tuple[tuple[str, str], ...]
+
+
+@dataclass(frozen=True, slots=True)
+class Hit:
+    location: str
+    form: str
+
+
+def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
+    for offset in range(3):
+        chunk: Final = raw[offset:]
+        yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
+
+
+def secret_forms(secret: str) -> Forms:
+    encoded: Final = (
+        ("url-encoded", urllib.parse.quote(secret, safe="")),
+        *_aligned_encodings(secret.encode(), "base64", base64.b64encode),
+        *_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
+    )
+    by_value: Final = MappingProxyType(
+        {value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
+    )
+    return (("raw", secret), *((name, value) for value, name in by_value.items()))
+
+
+def mask(text: str, forms: Forms) -> str:
+    return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
+
+
+def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
+    for form, value in forms:
+        if value in name:
+            yield Hit(mask(location, forms), f"{form} in name")
+
+
+def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
+    for form, value in forms:
+        if value.encode() in data:
+            yield Hit(mask(location, forms), form)
+    if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
+        return
+    with zipfile.ZipFile(io.BytesIO(data)) as archive:
+        for member in archive.infolist():
+            if member.is_dir():
+                continue
+            yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
+
+
+def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
+    for directory, subdirectories, files in os.walk(root):
+        yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
+        yield from ((Path(directory, name), False) for name in files)
+
+
+def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
+    location: Final = str(path.relative_to(root))
+    if linked_directory:
+        yield Hit(mask(location, forms), "directory symlink, not scanned")
+        return
+    yield from _name_hits(location, location, forms)
+    yield from _content_hits(location, path.read_bytes(), forms, 0)
+
+
+def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
+    forms: Final = secret_forms(secret)
+    entries: Final = tuple(_walk(root))
+    hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
+    return len(entries), hits
+
+
+def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
+    if len(argv) != 1:
+        print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
+        return 2
+    secret: Final = environ.get(SECRET_VARIABLE, "")
+    root: Final = Path(argv[0])
+    if not secret:
+        print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
+        return 0
+    if not root.is_dir():
+        print(f"{root} is not a directory, nothing to scan", file=out)
+        return 0
+    try:
+        scanned, hits = scan(root, secret)
+    except Exception as error:
+        print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
+        return 1
+    for hit in hits:
+        print(f"secret found: {hit.location} ({hit.form})", file=out)
+    if hits:
+        print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
+        return 1
+    print(f"no copy of the secret in {scanned} files under {root}", file=out)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

.github/workflows/lens-install-smoke.yml

@@ -1,152 +0,0 @@
-name: Lens installation smoke
-
-on:
-  workflow_dispatch:
-    inputs:
-      lens_image:
-        description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      lens_upgrade_image:
-        description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      gateway_baseline_ref:
-        description: Previously qualified gateway source commit, full SHA
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: lens-install-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-images:
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-          - component: backend
-            dockerfile: backend/Dockerfile
-          - component: ui
-            dockerfile: ui/Dockerfile
-          - component: migrations
-            dockerfile: migrations/Dockerfile
-          - component: monolith
-            dockerfile: Dockerfile
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-            baseline: true
-          - component: backend
-            dockerfile: backend/Dockerfile
-            baseline: true
-          - component: monolith
-            dockerfile: Dockerfile
-            baseline: true
-    env:
-      COMPONENT: ${{ matrix.component }}
-      DOCKERFILE: ${{ matrix.dockerfile }}
-      SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
-      IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
-      CURRENT_REF: ${{ github.sha }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ env.SOURCE_REF }}
-      - name: Verify the selected source
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          test "$(git rev-parse HEAD)" = "$SOURCE_REF"
-          if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
-            test "$SOURCE_REF" != "$CURRENT_REF"
-          fi
-      - name: Build the matching release image
-        run: |
-          docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-            -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
-      - name: Save the matching release image
-        run: |
-          docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
-            | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
-          path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
-          compression-level: 0
-          retention-days: 3
-          if-no-files-found: error
-          overwrite: true
-
-  helm-install:
-    needs: build-images
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-install-*-${{ github.sha }}
-          merge-multiple: true
-          path: ${{ runner.temp }}/lens-install-images
-      - name: Load the matching release images
-        run: |
-          for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
-            gzip -dc "$archive" | docker load
-            rm "$archive"
-          done
-      - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
-        with:
-          cosign-release: v2.4.3
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ghcr.io
-          username: ${{ vars.LENS_READ_USER || github.actor }}
-          password: ${{ secrets.LENS_READ_TOKEN }}
-      - name: Verify independent Lens images before loading them
-        env:
-          LENS_IMAGE: ${{ inputs.lens_image }}
-          LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
-          LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
-          LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
-          printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
-          echo "$LENS_PUBLIC_KEY_SHA256  $RUNNER_TEMP/lens.pub" | sha256sum --check
-          for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
-            [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
-            cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
-            cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
-            docker pull "$image"
-          done
-          docker tag "$LENS_IMAGE" lens-ci-worker:baseline
-          docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
-      - name: Install pinned Kubernetes test tools
-        run: |
-          curl --fail --location --output "$RUNNER_TEMP/kind" \
-            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
-          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kind"
-          curl --fail --location --output "$RUNNER_TEMP/kubectl" \
-            https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
-          echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea  $RUNNER_TEMP/kubectl" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kubectl"
-          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
-            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
-          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
-          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
-          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
-      - name: Install, ingest, upgrade, and restart both charts
-        run: bash tests/e2e/migrations/lens_helm_smoke.sh

.github/workflows/osv-scan.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - "litellm_**"
+      - "rc/**"
+      - "stable/**"
+      - "lts/**"
   schedule:
     - cron: "23 6 * * *"
   workflow_dispatch:

.github/workflows/test-integration.yml

@@ -0,0 +1,199 @@
+name: integration
+
+on:
+  pull_request:
+    types: [labeled]
+  schedule:
+    - cron: "5 8-20/3 * * 1-6"
+      timezone: America/Los_Angeles
+  workflow_dispatch:
+    inputs:
+      suite:
+        description: Integration group to run
+        type: choice
+        options:
+          - all
+          - providers
+          - extensions
+          - accounting
+          - management
+          - management-replica
+          - database
+          - database-replica
+          - mcp
+          - security
+          - cost
+          - sdk
+          - browser
+        default: all
+      workers:
+        description: Workers per group, blank keeps each group's default
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  integration:
+    name: integration ${{ matrix.suite }}
+    if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
+    environment: integration-license
+    runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
+    env:
+      UV_PYTHON: "3.12"
+      INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: circle_test
+        ports:
+          - 5432:5432
+        options: >-
+          --shm-size 1g
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+      redis:
+        image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Cache Prisma binaries
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Install redis-server 7.2.16
+        timeout-minutes: 3
+        run: |
+          mkdir -p "$RUNNER_TEMP/redis-bin"
+          cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
+          docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
+          docker rm "$cid"
+          "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
+          echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
+
+      - name: Install MCP SDK1 peer
+        if: matrix.suite == 'mcp'
+        timeout-minutes: 5
+        run: |
+          uv venv --python 3.12 .venv-mcp-peer
+          uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+          echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
+
+      - name: Set up Node.js
+        if: matrix.suite == 'browser'
+        timeout-minutes: 5
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version-file: ui/litellm-dashboard/.nvmrc
+          cache: "npm"
+          cache-dependency-path: |
+            ui/litellm-dashboard/package-lock.json
+            tests/e2e/ui/package-lock.json
+
+      - name: Install browser dependencies and build the dashboard
+        if: matrix.suite == 'browser'
+        timeout-minutes: 20
+        run: |
+          (cd ui/litellm-dashboard && npm ci)
+          (cd tests/e2e/ui && npm ci)
+          sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
+            node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
+          node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
+          (cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
+
+      - name: Run integration contracts
+        timeout-minutes: 45
+        env:
+          SUITE: ${{ matrix.suite }}
+          INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
+          INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
+          LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
+        run: |
+          case "$SUITE" in
+            *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
+            *) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
+          esac
+
+      - name: Verify every case ran exactly once
+        if: ${{ !cancelled() && matrix.suite != 'browser' }}
+        timeout-minutes: 2
+        run: |
+          record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
+          test -n "$record"
+          mkdir -p "$RUNNER_TEMP/shards/0"
+          cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
+          .venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
+
+      - name: Report MCP coverage
+        if: ${{ !cancelled() && matrix.suite == 'mcp' }}
+        timeout-minutes: 5
+        run: |
+          coverage_dir=test-results/integration-mcp/coverage
+          test -f "$coverage_dir/coverage.txt" && exit 0
+          export COVERAGE_FILE="$PWD/$coverage_dir/data"
+          .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
+          .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
+          .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
+          tail -n 1 "$coverage_dir/coverage.txt"
+
+      - name: Check results for the license
+        id: secret-scan
+        if: ${{ !cancelled() }}
+        timeout-minutes: 5
+        env:
+          SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
+        run: python .github/scripts/assert_secret_absent.py test-results
+
+      - name: Upload results
+        if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
+        timeout-minutes: 10
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: test-results/
+          retention-days: 14

.github/workflows/test-unit.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - "litellm_**"
+      - "rc/**"
+      - "stable/**"
+      - "lts/**"
   push:
     branches:
       - main
@@ -658,6 +661,7 @@ jobs:
           - shard: proxy-extras
             test-path: >-
               tests/unit/litellm_proxy_extras
+              tests/unit/db_scripts
             python-version: "3.12"
             workers: 2
             reruns: 0

.grype.yaml

@@ -11,3 +11,12 @@ ignore:
     package:
       name: zlib
       type: apk
+  - vulnerability: GHSA-v4x8-3wr5-3wq8
+    reason: >-
+      filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
+      the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
+      and the image then moves off 3.32.7, which retires this version-scoped rule
+    package:
+      name: filelock
+      version: 3.32.7
+      type: python

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -61,6 +61,7 @@ RUN apk add --no-cache \
     python-3.13-dev \
     rust \
     openssl \
+    libssl3 \
     openssl-dev \
     nodejs \
     npm \
@@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
 FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
 COPY --from=uvbin /uv /usr/local/bin/uv
 RUN apk add --no-cache python-3.13
-ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
-    https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
+ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
+    https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
 RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
     uv venv /opt/liteadmin --python python3.13 && \
     uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@@ -141,7 +142,7 @@ USER root
 RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app

README.md

@@ -615,15 +615,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
 
 ### Verify Docker Image Signatures
 
-All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
+All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
 
 **Verify using the pinned commit hash (recommended):**
 
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
+A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
 
 ```bash
 cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
+  --key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
   ghcr.io/berriai/litellm:<release-tag>
 ```
 

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -79,7 +79,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -94,7 +94,12 @@ ENV HOME=/home/nonroot \
     PYTHONUNBUFFERED=1 \
     PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
-COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
+COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
+COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
+COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
+COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
+COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

ci_cd/cost_map_guard.py

@@ -1,11 +1,11 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the three cost map files gets the file
-checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
-map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
-and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
-(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
-files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files gets the file
+checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
+skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
+branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
+get the file checks and additionally may only touch those two files or the legacy schema
+path and may only add or update models.
 """
 
 from __future__ import annotations
@@ -19,13 +19,12 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
-from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
-
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
-SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
-GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
+LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
+GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
@@ -35,7 +34,6 @@ CostMap = dict[str, object]
 class Snapshot:
     cost_map: str
     backup: str
-    schema: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
     return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
 
 
-def _rendered_schema(cost_map: CostMap) -> str:
-    try:
-        return render(build_schema(cost_map))
-    except SystemExit as error:
-        return str(error)
-
-
 def _evaluation_output_price_failure(
     key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
 ) -> str | None:
@@ -78,29 +69,13 @@ def _file_failures(
     *,
     evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
 ) -> tuple[str, ...]:
-    schema_text: Final = _rendered_schema(head_map)
-    if not schema_text.startswith("{"):
-        return (schema_text,)
     backup_failure: Final = (
         ()
         if head.backup == head.cost_map
         else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
     )
-    schema_failure: Final = (
-        ()
-        if head.schema == schema_text
-        else (
-            f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
-            "run `python ci_cd/generate_model_prices_schema.py` and commit the result",
-        )
-    )
     return (
         *backup_failure,
-        *schema_failure,
-        *(
-            f"{COST_MAP_PATH} does not validate against its schema: {error}"
-            for error in validation_errors(head_map, json.loads(schema_text))[:20]
-        ),
         *(
             failure
             for key, entry in head_map.items()
@@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
         *(
             f"bot PRs may only change the cost map files, not {path}"
             for path in changed_files
-            if path not in GUARDED_PATHS
+            if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
         ),
         *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
         *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

ci_cd/generate_model_prices_schema.py

@@ -1,418 +0,0 @@
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-from typing import Final, Optional
-
-import jsonschema
-
-REPO_ROOT = Path(__file__).parent.parent
-PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
-SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
-
-SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
-
-JsonSchema = dict
-
-NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
-NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
-BOOLEAN: JsonSchema = {"type": "boolean"}
-STRING: JsonSchema = {"type": "string"}
-TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
-WEEKDAY_PATTERN: Final = (
-    r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
-)
-
-EXTRA_BOOLEAN_KEYS = frozenset(
-    {
-        "gemini_native_audio",
-        "gemini_audio_only_live",
-        "uses_embed_content",
-        "use_openai_responses_path",
-        "bedrock_converse_supports_strict_tools",
-        "thinking_always_on",
-    }
-)
-
-HOURS_UTC: Final[JsonSchema] = {
-    "description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
-    "oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
-}
-
-OFF_PEAK_WINDOW: Final[JsonSchema] = {
-    "type": "object",
-    "properties": {
-        "hours_utc": HOURS_UTC,
-        "weekdays": {
-            "type": "array",
-            "description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
-            "items": {
-                "oneOf": [
-                    {"type": "integer", "minimum": 1, "maximum": 7},
-                    {"type": "string", "pattern": WEEKDAY_PATTERN},
-                ]
-            },
-            "minItems": 1,
-        },
-    },
-    "required": ["hours_utc"],
-    "additionalProperties": False,
-}
-
-OBJECT_KEYS: dict[str, JsonSchema] = {
-    "off_peak_pricing": {
-        "type": "object",
-        "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
-        "properties": {
-            "hours_utc": HOURS_UTC,
-            "windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
-            "weekday_timezone": {
-                "type": "string",
-                "description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
-            },
-            "input_cost_per_token": NONNEG_NUMBER,
-            "output_cost_per_token": NONNEG_NUMBER,
-            "output_cost_per_reasoning_token": NONNEG_NUMBER,
-            "cache_read_input_token_cost": NONNEG_NUMBER,
-            "cache_creation_input_token_cost": NONNEG_NUMBER,
-        },
-        "anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
-        "additionalProperties": False,
-    },
-    "search_context_cost_per_query": {
-        "type": "object",
-        "description": "USD cost per web search query, keyed by search context size.",
-        "properties": {
-            "search_context_size_low": NONNEG_NUMBER,
-            "search_context_size_medium": NONNEG_NUMBER,
-            "search_context_size_high": NONNEG_NUMBER,
-        },
-        "additionalProperties": False,
-    },
-    "guardrail_cost_per_unit": {
-        "type": "object",
-        "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
-        "additionalProperties": NONNEG_NUMBER,
-    },
-    "metadata": {
-        "type": "object",
-        "description": "Free-form notes about the entry (e.g. pricing derivation).",
-    },
-    "provider_specific_entry": {
-        "type": "object",
-        "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
-    },
-}
-
-ARRAY_KEYS: dict[str, JsonSchema] = {
-    "supported_audio_formats": {
-        "type": "array",
-        "description": "Audio container formats the model can return.",
-        "items": {"type": "string", "enum": ["mp3", "wav"]},
-    },
-    "supported_endpoints": {
-        "type": "array",
-        "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
-        "items": STRING,
-    },
-    "supported_modalities": {
-        "type": "array",
-        "description": "Input modalities the model accepts.",
-        "items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
-    },
-    "supported_output_modalities": {
-        "type": "array",
-        "description": "Output modalities the model can produce.",
-        "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
-    },
-    "reasoning_effort_levels": {
-        "type": "array",
-        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
-        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
-    },
-    "supported_regions": {
-        "type": "array",
-        "description": "Cloud regions the model is available in ('global' or region ids).",
-        "items": STRING,
-    },
-    "tiered_pricing": {
-        "type": "array",
-        "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
-        "items": {
-            "type": "object",
-            "properties": {
-                "range": {
-                    "type": "array",
-                    "description": "[min, max] prompt-token span this tier applies to.",
-                    "items": NONNEG_NUMBER,
-                    "minItems": 2,
-                    "maxItems": 2,
-                },
-                "max_results_range": {
-                    "type": "array",
-                    "description": "[min, max] result-count span this tier applies to (search models).",
-                    "items": NONNEG_NUMBER,
-                    "minItems": 2,
-                    "maxItems": 2,
-                },
-                "input_cost_per_token": NONNEG_NUMBER,
-                "output_cost_per_token": NONNEG_NUMBER,
-                "output_cost_per_reasoning_token": NONNEG_NUMBER,
-                "cache_read_input_token_cost": NONNEG_NUMBER,
-                "cache_creation_input_token_cost": NONNEG_NUMBER,
-                "input_cost_per_query": NONNEG_NUMBER,
-            },
-            "additionalProperties": False,
-        },
-    },
-}
-
-INTEGER_KEYS: dict[str, JsonSchema] = {
-    "max_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
-    },
-    "max_input_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Maximum prompt/context tokens the model accepts.",
-    },
-    "max_output_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Maximum tokens the model can generate in one response.",
-    },
-    "output_vector_size": {
-        **NONNEG_INTEGER,
-        "description": "Embedding dimension for embedding models.",
-    },
-    "prompt_cache_min_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
-    },
-    "token_count_framing": {
-        **NONNEG_INTEGER,
-        "description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
-    },
-    "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
-    "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
-}
-
-NUMBER_KEYS: dict[str, JsonSchema] = {
-    "regional_processing_uplift_multiplier_eu": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
-    },
-    "regional_processing_uplift_multiplier_us": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
-    },
-    "regional_endpoint_uplift_multiplier": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
-    },
-}
-
-COST_DESCRIPTIONS: dict[str, str] = {
-    "input_cost_per_token": "USD per prompt token.",
-    "output_cost_per_token": "USD per generated token.",
-    "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
-    "google_maps_grounding_cost_per_query": (
-        "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
-    ),
-    "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
-    "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
-    "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
-    "output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
-}
-
-
-def cost_description(key: str) -> Optional[str]:
-    if key in COST_DESCRIPTIONS:
-        return COST_DESCRIPTIONS[key]
-    if key.endswith("_flex"):
-        return "Flex service-tier rate for the same-named base field."
-    if key.endswith("_priority"):
-        return "Priority service-tier rate for the same-named base field."
-    if "_above_" in key:
-        return "Rate applied once the prompt exceeds the token threshold in the field name."
-    return None
-
-
-def cost_schema(key: str) -> JsonSchema:
-    description = cost_description(key)
-    return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
-
-
-def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
-    return {
-        "litellm_provider": {
-            "type": "string",
-            "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
-        },
-        "mode": {
-            "type": "string",
-            "description": "Primary API surface / task type of the model.",
-            "enum": list(modes),
-        },
-        "source": {
-            "type": "string",
-            "description": "URL of the provider pricing/model page this entry was taken from.",
-        },
-        "deprecation_date": {
-            "type": "string",
-            "description": "Date the provider deprecates the model, YYYY-MM-DD.",
-            "format": "date",
-            "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
-        },
-        "web_search_billing_unit": {
-            "type": "string",
-            "description": "Whether web search is billed per query or per prompt.",
-            "enum": ["per_query", "per_prompt"],
-        },
-        "bedrock_output_config_effort_ceiling": {
-            "type": "string",
-            "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
-            "enum": ["low", "medium", "high", "max", "xhigh"],
-        },
-        "default_reasoning_effort": {
-            "type": "string",
-            "description": (
-                "Reasoning effort the provider applies when the request omits reasoning_effort. "
-                "Gates whether a non-default temperature or the top_p/logprobs sampling params are "
-                "accepted, which hold only when the effort resolves to 'none'."
-            ),
-            "enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
-        },
-        "comment": STRING,
-        "token_count_fallback_model": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Model that counts tokens for this one when the provider cannot count it.",
-        },
-        "audio_transcription_config": STRING,
-        "vertex_ai_audio_api": {
-            "type": "string",
-            "enum": ["lyria_predict", "lyria_interactions"],
-        },
-    }
-
-
-def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
-    curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
-    if key in curated:
-        return curated[key]
-    if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
-        return BOOLEAN
-    if "cost" in key:
-        return cost_schema(key)
-    return None
-
-
-def build_schema(prices: dict) -> JsonSchema:
-    entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
-    all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
-    modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
-    unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
-    if unclassified:
-        raise SystemExit(
-            f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
-            f"Add them to the key tables in {Path(__file__).name} and rerun it."
-        )
-    entry_properties = {key: classify(key, modes) for key in all_keys}
-    return {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "title": "LiteLLM model_prices_and_context_window.json",
-        "description": (
-            "Schema for LiteLLM's model price and context window registry "
-            "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
-            "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
-            "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
-            "All costs are USD per unit. New optional fields are added regularly, so consumers should "
-            "ignore unknown fields rather than reject them."
-        ),
-        "type": "object",
-        "properties": {
-            "sample_spec": {
-                "type": "object",
-                "description": (
-                    "Documentation placeholder illustrating the entry shape; not a real model and not "
-                    "schema-conformant (several values are prose)."
-                ),
-            },
-            "fallback_generalizations": {
-                "type": "object",
-                "description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
-                "properties": {
-                    "rules": {
-                        "type": "array",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "name": STRING,
-                                "pattern": STRING,
-                                "description": STRING,
-                            },
-                            "required": ["name", "pattern"],
-                            "additionalProperties": True,
-                        },
-                    }
-                },
-                "additionalProperties": False,
-            },
-        },
-        "additionalProperties": {"$ref": "#/$defs/modelEntry"},
-        "$defs": {
-            "modelEntry": {
-                "type": "object",
-                "description": (
-                    "Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
-                    "are optional; boolean capability flags are simply omitted when unknown or false."
-                ),
-                "required": ["litellm_provider"],
-                "properties": entry_properties,
-                "additionalProperties": True,
-            }
-        },
-    }
-
-
-def render(schema: JsonSchema) -> str:
-    return json.dumps(schema, indent=2) + "\n"
-
-
-def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
-    validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
-    return tuple(
-        f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
-        for error in validator.iter_errors(prices)
-    )
-
-
-def main() -> int:
-    check = "--check" in sys.argv[1:]
-    prices = json.loads(PRICES_PATH.read_text())
-    rendered = render(build_schema(prices))
-    errors = validation_errors(prices, json.loads(rendered))
-    if errors:
-        print(f"{PRICES_PATH.name} does not validate against the generated schema:")
-        print("\n".join(errors[:20]))
-        return 1
-    if not check:
-        SCHEMA_PATH.write_text(rendered)
-        print(f"wrote {SCHEMA_PATH}")
-        return 0
-    if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
-        print(
-            f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
-            f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
-        )
-        return 1
-    print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

cosign.pub

@@ -1,4 +1,4 @@
 -----BEGIN PUBLIC KEY-----
-MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
-POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
+JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
 -----END PUBLIC KEY-----

db_scripts/strip_vector_store_internal_metadata.py

@@ -0,0 +1,201 @@
+import argparse
+import os
+import sys
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, repeat, takewhile
+from typing import Final, Literal, TextIO
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
+
+from litellm.vector_stores.utils import is_litellm_written_metadata_key
+
+DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+LIST_PAGE_SIZE: Final = "100"
+
+
+class VectorStore(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    metadata: dict[str, JsonValue] | None = None
+
+
+class VectorStorePage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    data: tuple[VectorStore, ...]
+    has_more: bool = False
+
+
+class OpenAIErrorBody(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    message: str = ""
+
+
+class OpenAIErrorResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    error: OpenAIErrorBody
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    api_key: str
+    api_base: str
+    vector_store_ids: tuple[str, ...]
+    extra_keys: frozenset[str]
+    apply: bool
+
+
+@dataclass(frozen=True, slots=True)
+class StoreResult:
+    store_id: str
+    matched_keys: tuple[str, ...]
+    status: Literal["dry_run", "updated", "failed"]
+    error: str = ""
+
+
+def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
+    return is_litellm_written_metadata_key(key) or key in extra_keys
+
+
+def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
+    return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
+
+
+def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
+    params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
+    return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
+
+
+def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
+    if page is None or not page.has_more or not page.data:
+        return None
+    return fetch_page(client, after=page.data[-1].id)
+
+
+def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
+    pages: Final = accumulate(
+        repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
+    )
+    for page in takewhile(lambda page: page is not None, pages):
+        yield from page.data
+
+
+def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
+    for store_id in store_ids:
+        yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
+
+
+def describe_http_error(response: httpx.Response) -> str:
+    try:
+        message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
+    except ValidationError:
+        return f"HTTP {response.status_code}"
+    return f"HTTP {response.status_code}: {message}"
+
+
+def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
+    try:
+        response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
+    except httpx.TransportError as error:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
+    if response.is_success:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
+    return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
+
+
+def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
+    keys: Final = matching_keys(store, options.extra_keys)
+    if not keys:
+        return None
+    if not options.apply:
+        return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
+    return delete_keys(client, store.id, keys)
+
+
+def format_result(result: StoreResult) -> str:
+    key_list: Final = ", ".join(result.matched_keys)
+    count: Final = len(result.matched_keys)
+    match result.status:
+        case "dry_run":
+            return f"{result.store_id}: would delete {count} keys: {key_list}"
+        case "updated":
+            return f"{result.store_id}: deleted {count} keys: {key_list}"
+        case "failed":
+            return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
+
+
+def clean_and_report(
+    client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
+) -> Iterator[StoreResult | None]:
+    for store in stores:
+        result: Final = clean_store(client, store, options)
+        if result is not None:
+            print(format_result(result), file=out)
+        yield result
+
+
+def run(options: Options, client: httpx.Client, out: TextIO) -> int:
+    stores: Final = (
+        retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
+    )
+    scanned: Final = tuple(clean_and_report(client, stores, options, out))
+    results: Final = tuple(result for result in scanned if result is not None)
+    updated: Final = sum(1 for result in results if result.status == "updated")
+    failed: Final = sum(1 for result in results if result.status == "failed")
+    mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
+    print(
+        f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
+        file=out,
+    )
+    return 1 if failed else 0
+
+
+def parse_options(argv: Sequence[str] | None) -> Options:
+    parser: Final = argparse.ArgumentParser(
+        description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
+    )
+    parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE)
+    parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
+    parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
+    parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
+    args: Final = parser.parse_args(argv)
+    if not args.api_key:
+        parser.error("--api-key or OPENAI_API_KEY is required")
+    return Options(
+        api_key=args.api_key,
+        api_base=args.api_base.rstrip("/"),
+        vector_store_ids=tuple(args.vector_store_id),
+        extra_keys=frozenset(args.extra_key),
+        apply=args.apply,
+    )
+
+
+def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
+    options: Final = parse_options(argv)
+    with httpx.Client(
+        base_url=options.api_base,
+        headers={"Authorization": f"Bearer {options.api_key}"},
+        timeout=60.0,
+        transport=transport,
+    ) as client:
+        try:
+            return run(options, client, sys.stdout)
+        except httpx.HTTPStatusError as error:
+            sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
+            return 2
+        except httpx.TransportError as error:
+            sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
+            return 2
+        except ValidationError:
+            sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
+            return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -59,6 +59,7 @@ RUN apk add --no-cache \
     python-3.13 \
     python-3.13-dev \
     openssl \
+    libssl3 \
     openssl-dev \
     nodejs \
     npm \
@@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
         coreutils \
         curl \
         openssl \
+        libssl3 \
         libsndfile \
         nodejs \
         npm && break || sleep 5; \
@@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
     done
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 

docker/build_from_pip/Dockerfile.build_from_pip

@@ -30,7 +30,7 @@ RUN uv venv --python python && \
       "google-cloud-aiplatform==1.133.0" \
       "google-genai==1.37.0" \
       "anthropic[vertex]==0.84.0" \
-      "grpcio==1.78.0" \
+      "grpcio==1.84.1" \
       "prometheus-client==0.20.0" \
       "langfuse==2.59.7" \
       "opentelemetry-api==1.28.0" \

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,11 +3,14 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 import asyncio
+import json
+import re
 from collections.abc import Mapping, Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
+from pydantic import TypeAdapter, ValidationError
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -38,7 +41,13 @@ if TYPE_CHECKING:
     from litellm.types.utils import LiteLLMBatch
 
 
+def _error_names_resource(error: Exception, resource_id: str) -> bool:
+    return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
+
+
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
+RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
+_FILE_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
 
 PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
     "completed",
@@ -108,6 +117,11 @@ class CheckBatchCost:
         # the guaranteed-failing primary query on every subsequent cycle.
         self._has_batch_processed_column: bool = True
         self.batch_processed_support_confirmed: bool = False
+        self._poll_cursor: tuple[datetime, str] | None = None
+        self._poll_before: datetime = datetime.now(timezone.utc)
+        self._arrival_cursor: tuple[datetime, str] | None = None
+        self._arrival_before: datetime = self._poll_before
+        self._poll_arrivals: bool = False
         self._line_item_stores: Final[set[asyncio.Task[None]]] = set()
         self._line_item_store_turn: Final = asyncio.Lock()
         self._poll_idle: Final = asyncio.Event()
@@ -160,29 +174,9 @@ class CheckBatchCost:
                 f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
             )
 
-        if not self._has_batch_processed_column:
-            return
-
-        # A row already in a terminal status is never rewritten by the sweep above, so
-        # without this it keeps a poll-page slot forever and starves newer batches.
-        retired: Final = await _managed_object_table(self.prisma_client).update_many(
-            where={
-                "file_purpose": "batch",
-                "batch_processed": False,
-                "status": {"in": ["complete", "completed"]},
-                "created_at": {"lt": cutoff},
-            },
-            data={"batch_processed": True},
-        )
-        if retired > 0:
-            verbose_proxy_logger.warning(
-                f"CheckBatchCost: gave up on {retired} completed managed objects older than "
-                f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
-            )
-
     async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
         """Query batch jobs without the batch_processed filter (for older schemas)."""
-        return await _managed_object_table(self.prisma_client).find_many(
+        return await self._find_jobs(
             where={
                 "file_purpose": "batch",
                 "status": {
@@ -196,10 +190,59 @@ class CheckBatchCost:
                     ]
                 },
             },
-            take=MAX_OBJECTS_PER_POLL_CYCLE,
-            order={"created_at": "asc"},
         )
 
+    async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
+        arrivals: Final = self._poll_arrivals
+        page: Final = await self._find_job_page(where, arrivals=arrivals)
+        self._poll_arrivals = not arrivals if page else arrivals
+        if page:
+            return page
+        return await self._find_job_page(where, arrivals=not arrivals)
+
+    async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
+        cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
+        if cursor is None:
+            if arrivals:
+                self._arrival_before = datetime.now(timezone.utc)
+            else:
+                self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
+        before: Final = self._arrival_before if arrivals else self._poll_before
+        page: Final = await _managed_object_table(self.prisma_client).find_many(
+            where={
+                **where,
+                "created_at": {
+                    "lte": before,
+                    **({"gt": before - RECENT_BATCH_POLL_WINDOW} if arrivals else {}),
+                },
+                **(
+                    {
+                        "OR": [
+                            {"created_at": {"gt": cursor[0]}},
+                            {"created_at": cursor[0], "id": {"gt": cursor[1]}},
+                        ]
+                    }
+                    if cursor is not None
+                    else {}
+                ),
+            },
+            take=MAX_OBJECTS_PER_POLL_CYCLE,
+            order=[{"created_at": "asc"}, {"id": "asc"}],
+        )
+        next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
+        if arrivals:
+            self._arrival_cursor = next_cursor
+        else:
+            self._poll_cursor = next_cursor
+        return page
+
+    def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
+        return {
+            "id": job.id,
+            "updated_at": job.updated_at,
+            **({"batch_processed": False} if self._has_batch_processed_column else {}),
+        }
+
     async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
         """
         Take a row that can never be costed out of the poll page. Leaving it selectable
@@ -207,27 +250,22 @@ class CheckBatchCost:
         once enough such rows accumulate no newer batch is ever reached. Older schemas
         without batch_processed can only be excluded through the status filter.
         """
-        data: Final = (
-            {"batch_processed": True}
-            if self._has_batch_processed_column
-            else {"status": "stale_expired"}
-        )
+        data: Final = {"batch_processed": True} if self._has_batch_processed_column else {"status": "stale_expired"}
         try:
-            await _managed_object_table(self.prisma_client).update(
-                where={"id": job.id},
+            retired: Final = await _managed_object_table(self.prisma_client).update_many(
+                where=self._unprocessed_snapshot(job),
                 data=data,
             )
         except Exception as db_err:
-            verbose_proxy_logger.error(
-                f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
-            )
+            verbose_proxy_logger.error(f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}")
+            return
+        if retired == 0:
             return
         verbose_proxy_logger.warning(
-            f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
-            "so it will no longer be polled"
+            f"CheckBatchCost: job {job.id} can never be costed ({reason}), so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
+    async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -241,10 +279,16 @@ class CheckBatchCost:
         """
         if not self._has_batch_processed_column:
             return True
+        from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
+
         try:
             claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
-                data={"batch_processed": True},
+                data={
+                    "batch_processed": True,
+                    "status": response.status if response.status != "completed" else "complete",
+                    "file_object": postgres_storable_json(response),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -253,7 +297,9 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
+    async def _release_job_claim(
+        self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
+    ) -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -265,7 +311,11 @@ class CheckBatchCost:
         try:
             await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
-                data={"batch_processed": False},
+                data={
+                    "batch_processed": False,
+                    "status": original_status,
+                    "file_object": json.dumps(original_file_object),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -299,13 +349,13 @@ class CheckBatchCost:
 
         from litellm.exceptions import NotFoundError
 
-        return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
 
     def _batch_deployment_exists(self, model_id: str) -> bool:
         """A 404 only proves the batch is gone when it came from the batch's own
         deployment. Once that deployment leaves the router, default fallbacks can
         silently send the retrieve to a provider that never saw the batch, so its
-        404 must not retire the row; the staleness sweep bounds it instead."""
+        404 must not retire a potentially recoverable row."""
         return self.llm_router.get_deployment(model_id=model_id) is not None
 
     @staticmethod
@@ -320,7 +370,7 @@ class CheckBatchCost:
 
         if not output_file_id:
             return False
-        return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
 
     async def _finalize_unbilled_terminal_job(
         self, job: "ManagedObjectRow", response: "LiteLLMBatch"
@@ -342,18 +392,18 @@ class CheckBatchCost:
                 db_batch_object=job,
                 unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
             )
-            update_data: Final[dict] = {
+            update_data: Final[dict[str, object]] = {
                 "status": response.status,
                 "file_object": response.model_dump_json(),
                 **({"batch_processed": True} if self._has_batch_processed_column else {}),
             }
-            await _managed_object_table(self.prisma_client).update(
-                where={"id": job.id},
+            finalized: Final = await _managed_object_table(self.prisma_client).update_many(
+                where=self._unprocessed_snapshot(job),
                 data=update_data,
             )
-            verbose_proxy_logger.info(
-                f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
-            )
+            if finalized == 0:
+                return False
+            verbose_proxy_logger.info(f"CheckBatchCost: marked job {job.id} as {response.status} in DB")
             return True
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -452,6 +502,10 @@ class CheckBatchCost:
         llm_provider: str,
         bare_model_name: str,
     ) -> Optional[Tuple[str, str]]:
+        stored_model_id: Final = self._get_stored_model_id(job)
+        if stored_model_id is not None and self._batch_deployment_exists(stored_model_id):
+            return stored_model_id, job.unified_object_id
+
         deployment_id = self._get_deployment_id_for_bare_model(bare_model_name, llm_provider)
         if deployment_id is None:
             verbose_proxy_logger.info(
@@ -563,25 +617,36 @@ class CheckBatchCost:
         )
 
     @staticmethod
-    def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
-        import json
-
-        from litellm.types.utils import LiteLLMBatch
-
+    def _parse_file_object(job: "ManagedObjectRow") -> Optional[dict[str, object]]:
         file_object: Final = job.file_object
         try:
-            parsed_file_object: Final[object] = (
+            return _FILE_OBJECT_ADAPTER.validate_python(
                 json.loads(file_object) if isinstance(file_object, str) else file_object
             )
-        except (json.JSONDecodeError, ValueError):
+        except (json.JSONDecodeError, ValueError, ValidationError):
             return None
-        if not isinstance(parsed_file_object, dict):
+
+    @classmethod
+    def _get_input_file_id(cls, job: "ManagedObjectRow") -> Optional[str]:
+        from litellm.types.utils import LiteLLMBatch
+
+        parsed_file_object: Final = cls._parse_file_object(job)
+        if parsed_file_object is None:
             return None
         try:
             return LiteLLMBatch.model_validate(parsed_file_object).input_file_id
         except Exception:
             return None
 
+    @classmethod
+    def _get_stored_model_id(cls, job: "ManagedObjectRow") -> Optional[str]:
+        """The deployment that created the batch, written into file_object by the managed
+        files hook at create time. Older rows have no such key and fall back to matching
+        the bare model name against the router."""
+        parsed_file_object: Final = cls._parse_file_object(job)
+        model_id: Final = parsed_file_object.get("model_id") if parsed_file_object else None
+        return model_id if isinstance(model_id, str) and model_id else None
+
     async def _track_completed_batch_cost(
         self,
         job: "ManagedObjectRow",
@@ -608,7 +673,11 @@ class CheckBatchCost:
         from litellm.files.types import FileContentCallOptions
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
+        from litellm.litellm_core_utils.litellm_logging import (
+            deployment_pricing_model_info,
+            mask_api_base_credentials,
+            resolve_vertex_location_for_cost,
+        )
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -718,8 +787,7 @@ class CheckBatchCost:
                         )
                     except Exception as _e:
                         verbose_proxy_logger.warning(
-                            f"CheckBatchCost: failed to create managed file ID for "
-                            f"{_file_attr}={_raw_file_id!r}: {_e}"
+                            f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
         # Pass the deployment's router-registered pricing (litellm_params custom
@@ -738,6 +806,12 @@ class CheckBatchCost:
             custom_llm_provider=batch_file_provider,
             model_name=model_name,
             model_info=deployment_model_info,
+            vertex_location=resolve_vertex_location_for_cost(
+                custom_llm_provider=llm_provider,
+                litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
+                optional_params=None,
+                model=model_name,
+            ),
         )
         error_file_failed_requests: Final = await count_error_file_failed_requests(
             response,
@@ -755,8 +829,9 @@ class CheckBatchCost:
                 failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
+        logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
         logging_obj = LiteLLMLogging(
-            model=batch_result.models[0],
+            model=logging_model,
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -787,27 +862,35 @@ class CheckBatchCost:
             custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
-        if not await self._claim_job_for_costing(job):
+        original_status: Final = job.status
+        original_file_object: Final = job.file_object
+        if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
             verbose_proxy_logger.info(
                 f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
                 "in this window, so its cost is already being tracked there"
             )
             return None
 
-        try:
-            await logging_obj.async_success_handler(
-                result=response,
-                batch_cost=batch_result.cost,
-                batch_usage=batch_result.usage,
-                batch_models=batch_result.models,
-                batch_successful_requests=batch_result.successful_requests,
-                batch_failed_requests=batch_result.failed_requests,
-                batch_prompt_cost=batch_result.prompt_cost,
-                batch_completion_cost=batch_result.completion_cost,
-            )
-        except Exception:
-            await self._release_job_claim(job)
-            raise
+        from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
+
+        with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
+            try:
+                await logging_obj.async_success_handler(
+                    result=response,
+                    batch_cost=batch_result.cost,
+                    batch_usage=batch_result.usage,
+                    batch_models=batch_result.models,
+                    batch_successful_requests=batch_result.successful_requests,
+                    batch_failed_requests=batch_result.failed_requests,
+                    batch_prompt_cost=batch_result.prompt_cost,
+                    batch_completion_cost=batch_result.completion_cost,
+                )
+            finally:
+                if accounting.retryable_failure:
+                    await self._release_job_claim(job, original_status, original_file_object)
+
+        if accounting.retryable_failure:
+            return None
 
         # Record batch duration (completed_at - created_at)
         if prom_logger and response.completed_at and response.created_at:
@@ -876,7 +959,10 @@ class CheckBatchCost:
         )
         from litellm.batches.batch_utils import fetch_batch_managed_file_content
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
+        from litellm.litellm_core_utils.litellm_logging import (
+            deployment_pricing_model_info,
+            resolve_vertex_location_for_cost,
+        )
 
         try:
             deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
@@ -920,6 +1006,12 @@ class CheckBatchCost:
                 cost_provider=cost_provider,
                 model_info=deployment_pricing_model_info(model_id=model_id, deployment_model=litellm_model_name),
                 metadata=line_item_metadata(await self._creator_attribution.batch_metadata(job, batch_id)),
+                vertex_location=resolve_vertex_location_for_cost(
+                    custom_llm_provider=llm_provider,
+                    litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
+                    optional_params=None,
+                    model=model_name,
+                ),
             )
             input_content: Final = (
                 b""
@@ -1015,6 +1107,7 @@ class CheckBatchCost:
         """
         try:
             from litellm.integrations.prometheus import PrometheusLogger
+
             prom_logger = PrometheusLogger.get_instance()
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not get Prometheus logger: {e}")
@@ -1039,7 +1132,7 @@ class CheckBatchCost:
                 # endpoint may transition a batch to "complete" before
                 # CheckBatchCost runs.  The batch_processed=False filter
                 # already prevents reprocessing finished batches.
-                jobs = await _managed_object_table(self.prisma_client).find_many(
+                jobs = await self._find_jobs(
                     where={
                         "file_purpose": "batch",
                         "batch_processed": False,
@@ -1052,8 +1145,6 @@ class CheckBatchCost:
                             ]
                         },
                     },
-                    take=MAX_OBJECTS_PER_POLL_CYCLE,
-                    order={"created_at": "asc"},
                 )
                 self.batch_processed_support_confirmed = True
             except Exception as query_err:
@@ -1099,10 +1190,7 @@ class CheckBatchCost:
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
-            if (
-                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
-                and response.output_file_id is not None
-            ):
+            if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
                 try:
                     tracked = await self._track_completed_batch_cost(
                         job=job,
@@ -1123,7 +1211,7 @@ class CheckBatchCost:
                         continue
                     verbose_proxy_logger.error(
                         f"CheckBatchCost: failed to track cost for batch {batch_id} "
-                        f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
+                        f"(job {job.id}): {tracking_err}"
                     )
                     self._record_error(prom_logger, "cost_tracking_error")
                     continue
@@ -1133,24 +1221,19 @@ class CheckBatchCost:
                 # Track this job for the final metrics summary
                 processed_models.append(tracked)
 
-                # mark the job as complete
-                try:
-                    update_data: dict = {
-                        "status": response.status if response.status != "completed" else "complete",
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await _managed_object_table(self.prisma_client).update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                    )
-                    if not self._has_batch_processed_column:
-                        # nothing claimed the row, so it stays in the poll page and its lines wait for that retry
+                if not self._has_batch_processed_column:
+                    try:
+                        await _managed_object_table(self.prisma_client).update(
+                            where={"id": job.id},
+                            data={
+                                "status": response.status if response.status != "completed" else "complete",
+                                "file_object": response.model_dump_json(),
+                            },
+                        )
+                    except Exception as db_err:
+                        verbose_proxy_logger.error(
+                            f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
+                        )
                         continue
                 self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
 

enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from dataclasses import dataclass
+from datetime import datetime
 from typing import TYPE_CHECKING, Final, Protocol, cast
 
 from pydantic import TypeAdapter
@@ -27,6 +28,12 @@ if TYPE_CHECKING:
 
 
 class ManagedObjectRow(Protocol):
+    @property
+    def created_at(self) -> datetime: ...
+
+    @property
+    def updated_at(self) -> datetime: ...
+
     @property
     def id(self) -> str: ...
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -19,6 +19,7 @@ from typing import (
     TypedDict,
     Union,
     cast,
+    runtime_checkable,
 )
 from uuid import NAMESPACE_URL, uuid5
 
@@ -366,6 +367,12 @@ class _ManagedResourcePrismaClient(Protocol):
     def db(self) -> _ManagedResourceDatabase: ...
 
 
+@runtime_checkable
+class _WriterRoutedDatabase(Protocol):
+    @property
+    def writer(self) -> _ManagedResourceDatabase: ...
+
+
 class _SchedulerWithJobLookup(Protocol):
     def get_job(self, job_id: str) -> object: ...
 
@@ -398,6 +405,15 @@ def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _Manag
     return prisma_client.db.litellm_managedobjecttable
 
 
+def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
+    return prisma_client.db
+
+
+def _writer_database(db: _ManagedResourceDatabase) -> _ManagedResourceDatabase:
+    """Reads that feed a write must see the create that just happened; a lagging read replica may not"""
+    return db.writer if isinstance(db, _WriterRoutedDatabase) else db
+
+
 _JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 _POSTGRES_UNSTORABLE_CHARACTER: Final = "\x00"
 
@@ -416,7 +432,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
     return value
 
 
-def _postgres_storable_json(file_object: BaseModel) -> str:
+def postgres_storable_json(file_object: BaseModel) -> str:
     dumped: Final = file_object.model_dump_json()
     if "\\u0000" not in dumped:
         return dumped
@@ -431,6 +447,25 @@ def _without_model_output(
     )
 
 
+def _with_model_id(
+    file_object: LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse,
+    model_id: str | None,
+) -> LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse:
+    return file_object if model_id is None else file_object.model_copy(update={"model_id": model_id})
+
+
+async def _stored_model_id(managed_object_table: _ManagedObjectTableActions, unified_object_id: str) -> str | None:
+    row: Final = await managed_object_table.find_first(where={"unified_object_id": unified_object_id})
+    if row is None:
+        return None
+    try:
+        parsed: Final = _JSON_VALUE_ADAPTER.validate_python(_decode_json_blob(row.file_object))
+    except (json.JSONDecodeError, ValidationError):
+        return None
+    model_id: Final = parsed.get("model_id") if isinstance(parsed, dict) else None
+    return model_id if isinstance(model_id, str) and model_id else None
+
+
 def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]:
     hidden_params: Final = cast(  # cast-ok: _hidden_params is an untyped attribute the upload path sets
         "Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {}
@@ -599,9 +634,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         persist_attribution: bool = False,
         create_if_missing: bool = True,
         batch_processed: bool = False,
+        creating_model_id: str | None = None,
     ) -> None:
         """Persist a managed object row, caching it and upserting it in the DB.
 
+        creating_model_id is the router deployment that created the object. It is kept
+        inside the stored file_object JSON as `model_id`, because the raw provider id an
+        unmanaged batch stores as unified_object_id says nothing about which deployment
+        made it, and the cost poller needs that deployment to read the result back. A
+        later observe of the row (create_if_missing=False) keeps the stored value rather
+        than overwriting it with whichever deployment served the retrieve.
+
         persist_attribution is set only by the batch and background response creates,
         the callers that can speak for the creator; it gates the api_key, request_tags, and,
         for a response, end_user_id columns that the cost poll jobs bill against, so a later poll or
@@ -622,7 +665,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         It is written only in the upsert create branch.
         """
         verbose_logger.info(f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache")
-        stored_object: Final = _without_model_output(file_object)
+        managed_database: Final = _managed_database(self.prisma_client)
+        managed_object_table: Final = managed_database.litellm_managedobjecttable
+        preserved_model_id: Final = (
+            creating_model_id
+            if creating_model_id is not None or create_if_missing
+            else await _stored_model_id(
+                _writer_database(managed_database).litellm_managedobjecttable, unified_object_id
+            )
+        )
+        stored_object: Final = _with_model_id(_without_model_output(file_object), preserved_model_id)
         litellm_managed_object = LiteLLM_ManagedObjectTable(
             unified_object_id=unified_object_id,
             model_object_id=model_object_id,
@@ -650,11 +702,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
         # FIX: Update status and file_object on every operation to keep state in sync
         update_columns: Final = {
-            "file_object": _postgres_storable_json(stored_object),
+            "file_object": postgres_storable_json(stored_object),
             "status": file_object.status,
             "updated_by": user_api_key_dict.user_id,
         }
-        managed_object_table: Final = _managed_object_table(self.prisma_client)
         if not create_if_missing:
             await managed_object_table.update_many(
                 where={"unified_object_id": unified_object_id},
@@ -666,7 +717,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             data={
                 "create": {
                     "unified_object_id": unified_object_id,
-                    "file_object": _postgres_storable_json(stored_object),
+                    "file_object": postgres_storable_json(stored_object),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
@@ -702,7 +753,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "unified_object_id": response_id,
                     "model_object_id": decrypted_response_id(response_id),
                     "file_purpose": "response",
-                    "file_object": _postgres_storable_json(_without_model_output(response)),
+                    "file_object": postgres_storable_json(_without_model_output(response)),
                     "status": PENDING_RESPONSE_STATUSES[0],
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
@@ -2069,6 +2120,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
                 persist_attribution=is_batch_create,
                 create_if_missing=is_batch_create,
+                creating_model_id=model_id if is_batch_create else None,
             )
             if not is_batch_create:
                 await self.store_batch_output_file_ownership(

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.77"
+version = "0.1.79.post1"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.77"
+version = "0.1.79.post1"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
 ARG PGBOUNCER_VERSION=1.26.0
@@ -32,7 +32,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -95,7 +95,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
+      apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -110,7 +110,12 @@ ENV HOME=/home/nonroot \
     PYTHONUNBUFFERED=1 \
     PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
-COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
+COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
+COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
+COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
+COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
+COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 

helm/litellm-helm/Chart.lock

@@ -5,8 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
-generated: '2026-10-09T04:18:10.643228000Z'
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: '2026-10-30T02:13:11.505726000Z'

helm/litellm-helm/Chart.yaml

@@ -39,6 +39,3 @@ dependencies:
     version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm-helm/README.md

@@ -249,6 +249,4 @@ itself.
 
 ## Connect Lens
 
-`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render

helm/litellm-helm/templates/_helpers.tpl

@@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
   value: {{ .Values.collector.drainTimeoutSeconds | quote }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -402,95 +398,28 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.labels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
-{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm-helm/templates/ingress.yaml

@@ -1,8 +1,6 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "litellm.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- $lensMode := include "litellm.lens.mode" . -}}
-{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@@ -46,18 +44,18 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
+          {{- if and $.Values.lens.enabled $.Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
-                name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
+                name: {{ $.Values.lens.service.name }}
                 port:
-                  number: {{ $.Values.lensWorker.service.port }}
+                  number: {{ $.Values.lens.service.port }}
               {{- else }}
-              serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
-              servicePort: {{ $.Values.lensWorker.service.port }}
+              serviceName: {{ $.Values.lens.service.name }}
+              servicePort: {{ $.Values.lens.service.port }}
               {{- end }}
           {{- end }}
           {{- range .paths }}

helm/litellm-helm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm-helm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm-helm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm-helm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm-helm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm-helm/tests/lens_connection_tests.yaml

@@ -0,0 +1,128 @@
+suite: lens connection
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - ingress.yaml
+tests:
+  - it: connects the proxy to an external Lens
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service ahead of the host paths
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[0].path
+          value: /lens-ingest
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,84 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - deployment.yaml
-  - configmap-litellm.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest

helm/litellm-helm/tests/lens_modes_tests.yaml

@@ -1,187 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- ingress.yaml
-- configmap-litellm.yaml
-- deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm-helm/tests/lens_modes_validation_tests.yaml

@@ -1,52 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm-helm/tests/lens_saved_secrets_tests.yaml

@@ -1,59 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

helm/litellm-helm/tests/lens_service_tests.yaml

@@ -1,167 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects deployment.yaml to the shared Lens service
-  template: deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.hosts:
-    - host: gateway.example
-      paths:
-      - path: /
-        pathType: Prefix
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in deployment.yaml
-  template: deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference

helm/litellm-helm/tests/lens_setup_tests.yaml

@@ -1,162 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60

helm/litellm-helm/values.yaml

@@ -678,70 +678,15 @@ serviceMonitor:
     # - test-namespace
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

helm/litellm/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
-generated: '2026-10-09T04:18:10.642706000Z'

helm/litellm/Chart.yaml

@@ -6,8 +6,3 @@ version: 0.1.0
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
-
-dependencies:
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm/README.md

@@ -1,10 +1,8 @@
 # LiteLLM componentized chart
 
-This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
+This chart deploys separate gateway, backend and UI services
 
-Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
 

helm/litellm/templates/_helpers.tpl

@@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -635,93 +631,27 @@ shutdown drain window.
 {{- end -}}
 
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -166,14 +166,14 @@ spec:
     - host: {{ $componentHosts.gateway | quote }}
       http:
         paths:
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           - path: /
             pathType: Prefix
@@ -249,15 +249,15 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           {{- /*
             --- Operator-supplied extra paths (ingress.extraPaths) ---

helm/litellm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm/tests/component_hosts_tests.yaml

@@ -15,7 +15,6 @@ tests:
       ingress.componentHosts.ui: ui.example.com
       ingress.componentHosts.backend: backend.example.com
       ingress.componentHosts.gateway: gateway.example.com
-      lensWorker.mode: disabled
       ingress.tls:
         - hosts:
             - ui.example.com
@@ -83,12 +82,12 @@ tests:
       ui.backendUrl: https://backend.example.com
       ui.publicUrl: https://ui.example.com
       backend.gatewayUrl: https://gateway.example.com
-      lensWorker.mode: external
-      lensWorker.externalServiceName: existing-lens
-      lensWorker.externalUrl: https://lens.example.com
-      lensWorker.publicUrl: https://lens.example.com
-      lensWorker.serviceTokenSecret.name: existing-lens-signing
-      lensWorker.gateway.secretName: existing-lens-signing
+      lens.enabled: true
+      lens.service.name: existing-lens
+      lens.url: http://existing-lens:4318
+      lens.publicUrl: https://gateway.example.com/lens-ingest
+      lens.serviceTokenSecret.name: existing-lens-signing
+      lens.gatewaySecret.name: existing-lens-signing
     asserts:
       - equal:
           path: spec.rules[2].http.paths[0].path

helm/litellm/tests/lens_connection_tests.yaml

@@ -0,0 +1,166 @@
+suite: lens connection
+templates:
+  - gateway/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: connects the gateway to an external Lens
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: connects the backend to an external Lens
+    template: backend/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service on the single ingress host
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,86 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - gateway/deployment.yaml
-  - gateway/configmap.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.host: gateway.example
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_modes_tests.yaml

@@ -1,208 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-- ingress.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: keeps the ingestion endpoint on an independently deployed Lens service
-  template: ingress.yaml
-  set:
-    ingress.enabled: true
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: existing-lens
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm/tests/lens_modes_validation_tests.yaml

@@ -1,54 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm/tests/lens_saved_secrets_tests.yaml

@@ -1,61 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_service_tests.yaml

@@ -1,196 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- gateway/configmap.yaml
-- gateway/deployment.yaml
-- backend/deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects gateway/deployment.yaml to the shared Lens service
-  template: gateway/deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: connects backend/deployment.yaml to the shared Lens service
-  template: backend/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.host: gateway.example
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: gateway/deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in gateway/deployment.yaml
-  template: gateway/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: omits Lens connection settings when disabled in backend/deployment.yaml
-  template: backend/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: gateway/deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference
-values:
-- ./values/required.yaml

helm/litellm/tests/lens_setup_tests.yaml

@@ -1,164 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_worker_tests.yaml

@@ -1,79 +0,0 @@
-suite: Independent Lens release and credentials
-templates:
-  - lens/deployment.yaml
-values:
-  - ./values/required.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-  lensWorker.serviceTokenSecret.name: lens-service
-  lensWorker.clickhouseSecret.name: lens-storage
-tests:
-  - it: selects a Lens release independently of the gateway
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-      lensWorker.image.tag: 2.3.4
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:2.3.4
-  - it: retains the pinned Lens chart default when only the gateway changes
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:0.1.0-dev.0
-  - it: selects development publications explicitly
-    set:
-      lensWorker.image.repository: ghcr.io/berriai/lens-dev
-      lensWorker.image.tag: sha-0123456789abcdef
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
-  - it: preserves a verified registry mirror and digest
-    set:
-      lensWorker.image.repository: registry.example/lens
-      lensWorker.image.tag: replaced-label
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: rejects a malformed digest
-    set:
-      lensWorker.image.digest: sha256:invalid
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
-  - it: keeps the legacy enablement switch working
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments: {count: 0}
-  - it: keeps isolated local analysis with delegated gateway access
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content: {name: LENS_MODE, value: standalone}
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_GATEWAY_SECRET
-            valueFrom:
-              secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
-      - equal:
-          path: spec.template.spec.automountServiceAccountToken
-          value: false
-      - equal:
-          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
-          value: true
-      - equal:
-          path: spec.template.spec.volumes[0].emptyDir
-          value: {medium: Memory, sizeLimit: 1Gi}

helm/litellm/values.yaml

@@ -654,70 +654,15 @@ ui:
   topologySpreadConstraints: []
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.109"
+version = "0.4.110"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.109"
+version = "0.4.110"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -8201,18 +8201,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
 dependencies = [
  "rustls-pki-types",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
+checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
 dependencies = [
  "rustls-pki-types",
 ]

litellm-rust/Cargo.toml

@@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"
 percent-encoding = "2.3"
-webpki-roots = "1"
+webpki-roots = "1.0.10"
 time = { version = "0.3.53", features = ["parsing"] }
 criterion = "0.8.2"
 fancy-regex = "0.19.2"

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
     assert_eq!(sent, expected);
 }
 
+#[rstest]
+#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
+#[case::deepseek("deepseek", &[])]
+#[case::minimax("minimax", &[])]
+#[tokio::test]
+async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
+    call: MessagesCall,
+    #[case] provider: &str,
+    #[case] derived: &[AnthropicBeta],
+) {
+    let upstream = upstream([message_response()]).await;
+    let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": safeguards}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    let expected: BetaSet = derived
+        .iter()
+        .cloned()
+        .chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+        .collect();
+    assert_eq!(sent_betas(&request), expected);
+    assert_eq!(
+        request.body_json::<Value>().unwrap()["safeguards"],
+        safeguards
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
+    call: MessagesCall,
+    #[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": null}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    assert_eq!(
+        sent_betas(&request),
+        BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+    );
+    assert!(
+        !request
+            .body_json::<Map<String, Value>>()
+            .unwrap()
+            .contains_key("safeguards")
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -109,7 +109,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        merge_beta_headers(
+            headers,
+            provider_feature_betas(request, BetaProvider::Anthropic),
+        )
     }
 }
 
@@ -136,12 +139,20 @@ pub(crate) fn transform_messages_request(
         messages: strip_encrypted_reasoning_blocks(messages),
         params: MessagesOptionalParams {
             context_management,
+            extra: without_null_safeguards(request.params.extra),
             ..request.params
         },
         ..request
     })
 }
 
+fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
+    extra
+        .into_iter()
+        .filter(|(key, value)| key != "safeguards" || !value.is_null())
+        .collect()
+}
+
 pub(crate) fn update_headers_with_anthropic_beta(
     headers: Headers,
     request: &MessagesRequest,
@@ -149,12 +160,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
     merge_beta_headers(headers, feature_betas(request))
 }
 
-/// The betas a request needs on a host other than the first-party API: the Anthropic set
-/// filtered and renamed through the host's `BetaProvider` policy.
+pub(crate) fn update_headers_with_safeguards_beta(
+    headers: Headers,
+    request: &MessagesRequest,
+    provider: BetaProvider,
+) -> Headers {
+    merge_beta_headers(
+        headers,
+        feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
+    )
+}
+
+/// The betas a request needs on a host with a column in the beta map, filtered and renamed
+/// through that column.
 pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
-    feature_betas(request)
-        .iter()
-        .filter_map(|beta| beta.on(provider))
+    on_provider(
+        feature_betas(request).union(mapped_host_betas(&request.params)),
+        provider,
+    )
+}
+
+fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
+    betas.iter().filter_map(|beta| beta.on(provider)).collect()
+}
+
+fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
+    carries_safeguards(params)
+        .then_some(AnthropicBeta::DangerousToolUse20260903)
+        .into_iter()
         .collect()
 }
 
@@ -206,6 +239,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
             .is_some_and(|config| config.format.is_some())
 }
 
+fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
+    params
+        .extra
+        .get("safeguards")
+        .is_some_and(|value| !value.is_null())
+}
+
 fn messages_carry_output_config(messages: &[Message]) -> bool {
     messages
         .iter()
@@ -1025,10 +1065,44 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
         &["per-turn-control-2026-07-01"]
     )]
+    #[case::safeguards(
+        json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
+        &[]
+    )]
     fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
         assert_eq!(feature_betas(&request(fields)), betas(expected));
     }
 
+    #[rstest]
+    #[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
+    #[case::null(BetaProvider::Anthropic, Value::Null, &[])]
+    #[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
+    fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
+        #[case] provider: BetaProvider,
+        #[case] safeguards: Value,
+        #[case] expected: &[&str],
+    ) {
+        assert_eq!(
+            provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
+            betas(expected)
+        );
+    }
+
+    #[rstest]
+    fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
+        let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
+        assert_eq!(
+            update_headers_with_anthropic_beta(
+                client.clone(),
+                &request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
+            ),
+            client
+        );
+    }
+
     #[rstest]
     #[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
     #[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@@ -1067,6 +1141,11 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
         &[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
     )]
+    #[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
+        &[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
+    )]
     fn request_headers_merge_the_feature_betas(
         #[case] input: &[(&str, &str)],
         #[case] fields: Value,

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -2,7 +2,7 @@ use litellm_auth::SecretValue;
 use litellm_http::request::{has_bearer_auth, has_header};
 use litellm_llms_types::{
     formats::messages::MessagesRequest,
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 
@@ -12,7 +12,7 @@ use crate::{
         common_utils::API_KEY_PLACEMENT,
         messages::{
             handler::shape_anthropic_messages_request,
-            transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+            transformation::{transform_messages_request, update_headers_with_safeguards_beta},
         },
     },
     azure_ai::common_utils::{
@@ -104,7 +104,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
     }
 }
 
@@ -544,6 +544,12 @@ mod tests {
         &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
     )]
     #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
+    #[case::safeguards_need_the_dangerous_tool_use_beta(
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219")],
+        &[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
+    )]
+    #[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
     fn request_headers_carry_the_anthropic_feature_betas(
         #[case] fields: serde_json::Value,
         #[case] forwarded: &[(&str, &str)],

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -6,7 +6,7 @@ use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{header_value, with_header, without_headers};
 use litellm_llms_types::{
     formats::messages::MessagesRequest,
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
@@ -15,7 +15,7 @@ use crate::{
     Error, ErrorDetail,
     anthropic::messages::{
         handler::shape_anthropic_messages_request,
-        transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+        transformation::{transform_messages_request, update_headers_with_safeguards_beta},
     },
     base_llm::{
         auth::{AuthScheme, Headers, ValidatedEnvironment},
@@ -174,7 +174,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
             None => headers,
         };
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
     }
 
     fn wire_body(&self, body: Value) -> Value {
@@ -356,6 +356,21 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
+    #[case::null_safeguards(Value::Null, None)]
+    fn safeguards_need_the_dangerous_tool_use_beta(
+        #[case] safeguards: Value,
+        #[case] expected: Option<&str>,
+    ) {
+        let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
+        let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
+        assert_eq!(
+            header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
+            expected.map(str::to_string)
+        );
+    }
+
     #[rstest]
     #[case::default(json!({}), false)]
     #[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

litellm-rust/crates/llms/src/deepseek/messages/transformation.rs

@@ -451,4 +451,27 @@ mod tests {
             json!({"type": "enabled", "budget_tokens": 1024})
         );
     }
+
+    #[rstest]
+    #[case::without_client_betas(&[])]
+    #[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
+    fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
+        let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+        let body = json!({
+            "model": "deepseek-v4-pro",
+            "max_tokens": 100,
+            "messages": [{"role": "user", "content": "hi"}],
+            "safeguards": safeguards
+        });
+        let headers: Headers = client
+            .iter()
+            .map(|(name, value)| (name.to_string(), value.to_string()))
+            .collect();
+        assert_eq!(
+            DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
+                .request_headers(headers.clone(), &request_from(body.clone())),
+            headers
+        );
+        assert_eq!(transformed(body)["safeguards"], safeguards);
+    }
 }

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -213,23 +213,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
 }
 
 /// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
-/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
-/// out.
+/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
 fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
     let params = &request.params;
     provider_feature_betas(request, BetaProvider::VertexAi).union(
-        [
-            has_web_search_tool(params.tools.as_deref())
-                .then_some(AnthropicBeta::WebSearch20250305),
-            params
-                .extra
-                .get("safeguards")
-                .is_some_and(|value| !value.is_null())
-                .then_some(AnthropicBeta::DangerousToolUse20260903),
-        ]
-        .into_iter()
-        .flatten()
-        .collect(),
+        has_web_search_tool(params.tools.as_deref())
+            .then_some(AnthropicBeta::WebSearch20250305)
+            .into_iter()
+            .collect(),
     )
 }
 

litellm-rust/crates/model-catalog/AGENTS.md

@@ -2,5 +2,4 @@
 
 For `model_prices_and_context_window.json` validation, we should eventually:
 
-- Remove any schema file like `model_prices_and_context_window.schema.json`
 - Stop skipping this crate's tests

litellm/__init__.py

@@ -571,6 +571,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
 
 model_cost = get_model_cost_map(url=model_cost_map_url)
 cost_discount_config: Dict[str, float] = {}  # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
+apply_search_free_tier: bool = False
 cost_margin_config: Dict[
     str, Union[float, Dict[str, float]]
 ] = {}  # Provider-specific or global cost margins. Examples:
@@ -763,7 +764,7 @@ def is_openai_finetune_model(key: str) -> bool:
 
 
 def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
-    for key, value in model_cost_map.items():
+    for key, value in tuple(model_cost_map.items()):
         if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
             open_ai_chat_completion_models.add(key)
         elif value.get("litellm_provider") == "text-completion-openai":
@@ -1416,6 +1417,7 @@ from .exceptions import (
     InvalidRequestError,
     BadRequestError,
     ImageFetchError,
+    MalformedRequestFieldError as MalformedRequestFieldError,
     VectorStoreSearchError,
     NotFoundError,
     PermissionDeniedError,

litellm/anthropic_beta_headers_manager.py

@@ -22,17 +22,28 @@ Environment Variables:
 - LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
 """
 
+import asyncio
 import json
 import os
+import threading
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from functools import cache
 from importlib.resources import files
-from typing import Final
+from typing import Final, TypeAlias
 
 import httpx
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 
 # Cache for the loaded configuration
 _BETA_HEADERS_CONFIG: dict | None = None
+_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
+_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
+
+BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
+_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 
 
 class GetAnthropicBetaHeadersConfig:
@@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
     if _BETA_HEADERS_CONFIG is not None:
         return _BETA_HEADERS_CONFIG
 
-    # Get the URL from environment or use default
-    from litellm import anthropic_beta_headers_url
+    with _BETA_HEADERS_CONFIG_LOCK:
+        if _BETA_HEADERS_CONFIG is not None:
+            return _BETA_HEADERS_CONFIG
 
-    _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
-    verbose_logger.debug("Loaded and cached beta headers config")
+        # Get the URL from environment or use default
+        from litellm import anthropic_beta_headers_url
 
-    return _BETA_HEADERS_CONFIG
+        _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
+        verbose_logger.debug("Loaded and cached beta headers config")
+
+        return _BETA_HEADERS_CONFIG
+
+
+async def aload_beta_headers_config() -> None:
+    if _BETA_HEADERS_CONFIG is None:
+        await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
+
+
+@cache
+def _bundled_beta_headers_config() -> BetaHeadersConfig:
+    return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
 
 
 def reload_beta_headers_config() -> dict:
@@ -288,6 +313,14 @@ def is_beta_header_supported(
     return beta_header in provider_mapping and provider_mapping[beta_header] is not None
 
 
+def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    provider_mapping: Final = config.get(canonical_provider, {})
+    return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
+
+
 def get_provider_beta_header(
     anthropic_beta_header: str,
     provider: str,
@@ -321,6 +354,7 @@ def get_provider_beta_header(
 def update_headers_with_filtered_beta(
     headers: dict,
     provider: str,
+    keep: frozenset[str] = frozenset(),
 ) -> dict:
     """
     Update headers dict by filtering and transforming anthropic-beta header values.
@@ -329,6 +363,7 @@ def update_headers_with_filtered_beta(
     Args:
         headers: Request headers dict (will be modified in place)
         provider: Provider name
+        keep: Betas the routed model accepts, forwarded as sent whatever the row says
 
     Returns:
         Updated headers dict
@@ -338,12 +373,19 @@ def update_headers_with_filtered_beta(
         return headers
 
     # Parse existing beta headers
-    beta_values: Final = [b.strip() for b in existing_beta.split(",") if b.strip()]
+    beta_values: Final[tuple[str, ...]] = tuple(b.strip() for b in existing_beta.split(",") if b.strip())
 
     # Filter and transform based on provider
-    filtered_beta_values: Final = filter_and_transform_beta_headers(
-        beta_headers=beta_values,
-        provider=provider,
+    filtered_beta_values: Final = tuple(
+        dict.fromkeys(
+            [
+                *filter_and_transform_beta_headers(
+                    beta_headers=[b for b in beta_values if b not in keep],
+                    provider=provider,
+                ),
+                *(b for b in beta_values if b in keep),
+            ]
+        )
     )
 
     # Update or remove the header

litellm/batches/batch_line_items.py

@@ -38,6 +38,7 @@ class BatchLineItemSource:
     cost_provider: BatchFileProvider
     model_info: ModelInfo | None
     metadata: StandardBatchLineItemMetadata
+    vertex_location: str | None = None
 
 
 def line_item_metadata(attribution: Mapping[str, object]) -> StandardBatchLineItemMetadata:
@@ -140,7 +141,11 @@ def _line_item(
     source: BatchLineItemSource, custom_id: str | None, request: Mapping[str, object] | None, entry: dict[str, object]
 ) -> StandardBatchLineItem:
     priced: Final = batch_cost_and_usage(
-        (entry,), source.cost_provider, model_name=source.model, model_info=source.model_info
+        (entry,),
+        source.cost_provider,
+        model_name=source.model,
+        model_info=source.model_info,
+        vertex_location=source.vertex_location,
     )
     succeeded: Final = priced.successful_requests > 0
     return StandardBatchLineItem(

litellm/batches/batch_utils.py

@@ -77,9 +77,10 @@ async def calculate_batch_cost_and_usage(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Calculate the cost and usage of a batch, see batch_cost_and_usage."""
-    return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info)
+    return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info, vertex_location)
 
 
 def batch_cost_and_usage(
@@ -87,6 +88,7 @@ def batch_cost_and_usage(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Calculate the cost and usage of a batch.
@@ -96,16 +98,21 @@ def batch_cost_and_usage(
             pricing. Threaded through to batch_cost_calculator so that
             deployment-specific pricing (e.g. input_cost_per_token_batches)
             is used instead of the global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     first_row: Final = file_content_dictionary[0] if file_content_dictionary else None
     if _uses_native_vertex_output(custom_llm_provider, model_name, first_row):
-        return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info)
+        return calculate_vertex_ai_batch_cost_and_usage(
+            file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location
+        )
 
     return _aggregate_batch_cost_usage_models(
         entries=file_content_dictionary,
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -115,6 +122,7 @@ async def handle_completed_batch(
     model_name: str | None = None,
     litellm_params: dict[str, object] | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
@@ -128,6 +136,8 @@ async def handle_completed_batch(
         model_info: Optional deployment-level model info with custom pricing,
             threaded through so a deployment's configured rates win over the
             global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     # A completed batch whose request lines all failed has no output file - the
     # results are written to a separate error_file_id and output_file_id is None.
@@ -155,7 +165,10 @@ async def handle_completed_batch(
 
     output_file_result: Final = (
         calculate_vertex_ai_batch_cost_and_usage(
-            _iter_batch_output_entries(file_content), model_name, model_info=model_info
+            _iter_batch_output_entries(file_content),
+            model_name,
+            model_info=model_info,
+            vertex_location=vertex_location,
         )
         if _uses_native_vertex_output(
             custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None)
@@ -165,6 +178,7 @@ async def handle_completed_batch(
             custom_llm_provider=custom_llm_provider,
             model_name=model_name,
             model_info=model_info,
+            vertex_location=vertex_location,
         )
     )
 
@@ -203,6 +217,7 @@ def _classify_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
     """Classify every output line in a single pass, so counting failures never needs
     a second read of a potentially huge output file. A line the provider reported as
@@ -213,7 +228,7 @@ def _classify_output_line_stats(
         if not _batch_response_was_successful(entry, custom_llm_provider):
             yield _LineOutcome.PROVIDER_FAILED
             continue
-        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
         yield stats if stats is not None else _LineOutcome.UNCOSTABLE
 
 
@@ -222,13 +237,14 @@ def _safe_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats | None:
     """Return the stats for one provider-successful batch output line, or None when
     it cannot be costed, so a single bad line never aborts the whole batch's cost
     accounting."""
     custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
     try:
-        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
     except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
         verbose_logger.warning(
             "batch output line could not be costed, so it is billed at $0 and the rest of the batch "
@@ -244,6 +260,7 @@ def _compute_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats:
     response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
     usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
@@ -258,6 +275,7 @@ def _compute_output_line_stats(
         model_name=model_name,
         response_model=response_model,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
     return _BatchOutputLineStats(
         prompt_cost=line_prompt_cost,
@@ -287,6 +305,7 @@ def _output_line_cost(
     model_name: str | None,
     response_model: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """(prompt_cost, completion_cost) for one output line, priced at batch rates."""
     from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
@@ -307,6 +326,7 @@ def _output_line_cost(
         model=cost_model,
         custom_llm_provider=custom_llm_provider,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -315,11 +335,14 @@ def _aggregate_batch_cost_usage_models(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Aggregate cost, usage, models, and pass/fail counts from batch output
     entries in a single pass, holding one small stats record per line instead
     of the parsed file."""
-    all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+    all_results: Final = tuple(
+        _classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location)
+    )
     line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
     failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
     successful_requests: Final = len(all_results) - failed_requests
@@ -366,6 +389,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
     vertex_ai_batch_responses: Iterable[dict[str, object]],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Cost and usage of a native Vertex predictions.jsonl, one
@@ -384,6 +408,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
             model_info=model_info,
             calculate_usage=VertexGeminiConfig.calculate_usage,
             cost_calculator=batch_cost_calculator,
+            vertex_location=vertex_location,
         )
         for row in vertex_ai_batch_responses
     )
@@ -793,11 +818,17 @@ def _batch_response_was_successful(
     message batch results lines report ``result.type == "succeeded"``; Bedrock
     batch output lines report ``modelOutput`` (and no ``error``).
     """
-    if custom_llm_provider == "anthropic":
-        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
-        dict[str, object], batch_job_output_file.get("response", None) or {}
+    envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
+    envelope: Final = batch_job_output_file.get(envelope_key)
+    if not isinstance(envelope, Mapping):
+        if envelope is not None:
+            verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
+        return False
+    response: Final = cast(Mapping[object, object], envelope)  # cast-ok: mapping shape validated above
+    return (
+        response.get("type") == "succeeded"
+        if custom_llm_provider == "anthropic"
+        else response.get("status_code") == 200
     )
-    return _response.get("status_code", None) == 200

litellm/caching/caching_handler.py

@@ -512,7 +512,7 @@ class LLMCachingHandler:
                     )
 
                     if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
-                        logging_obj.handle_sync_success_callbacks_for_async_calls(
+                        logging_obj.handle_sync_cache_hit_success_callbacks(
                             result=cached_result,
                             start_time=start_time,
                             end_time=end_time,

litellm/caching/dual_cache.py

@@ -59,16 +59,6 @@ class PendingBatchRead:
     previous_access_times: dict[str, float | None]
 
 
-@dataclass(frozen=True, slots=True)
-class DeclaredBatchRead:
-    """A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
-    so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
-
-    keys: tuple[str, ...]
-    pending: PendingBatchRead
-    result: BatchResult[Mapping[str, object]] | None
-
-
 def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
     if future.cancelled():
         return
@@ -382,22 +372,6 @@ class DualCache(BaseCache):
                     await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
         return merged
 
-    async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
-        pending: Final = await self._prepare_batch_get(
-            list(keys),
-            local_only=False,
-            throttle_redis=False,
-        )
-        return DeclaredBatchRead(
-            keys=tuple(keys),
-            pending=pending,
-            result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
-        )
-
-    async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
-        redis_result: Final = None if declared.result is None else await declared.result
-        return await self._apply_batch_get(declared.pending, redis_result)
-
     async def async_batch_get_cache(
         self,
         keys: list,

litellm/caching/evicted_client_closer.py

@@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
 close them, and each bucket is ordered by deadline, so a reap walks the entries
 that are due rather than the whole queue.
 
+A bucket whose loop has been collected is dropped, and its clients are left to the
+collector, as they are when the queue is full.
+
 The queue holds its clients weakly, so waiting out a grace window never keeps
 alive anything the collector would have reclaimed first.
 """
@@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
 import asyncio
 import contextlib
 import inspect
+import itertools
 import threading
 import time
 import weakref
@@ -53,6 +57,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
 _CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
 
 _BucketKey = str | int
+_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
+_LOOP_TOKEN_COUNTER: Final = itertools.count()
+_LOOP_TOKEN_LOCK: Final = threading.Lock()
 
 
 @dataclass(frozen=True, slots=True)
@@ -63,12 +70,12 @@ class _PendingClose:
     collector would otherwise have reclaimed first.
 
     ``needs_loop`` is set for a client whose close is a coroutine; those can only
-    be closed from the event loop they were evicted on, recorded in ``loop_id``.
+    be closed from the event loop they were evicted on, recorded in ``loop_token``.
     A client that closes synchronously carries neither constraint.
     """
 
     client_ref: "weakref.ref[object]"
-    loop_id: int | None
+    loop_token: int | None
     needs_loop: bool
     close_after: float
 
@@ -77,16 +84,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
     """Which reaps can close this entry: any at all, any running a loop, or one loop's."""
     if not pending.needs_loop:
         return _CLOSABLE_ANYWHERE
-    if pending.loop_id is None:
+    if pending.loop_token is None:
         return _CLOSABLE_ON_ANY_LOOP
-    return pending.loop_id
+    return pending.loop_token
 
 
-def _running_loop_id() -> int | None:
+def running_loop_token() -> int | None:
     try:
-        return id(asyncio.get_running_loop())
+        loop: Final = asyncio.get_running_loop()
     except RuntimeError:
         return None
+    try:
+        token: Final[int | None] = _LOOP_TOKENS.get(loop)
+        if token is not None:
+            return token
+        with _LOOP_TOKEN_LOCK:
+            return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
+    except TypeError:
+        return id(loop)
 
 
 def _close_function(client: object) -> Callable[[], object] | None:
@@ -167,6 +182,8 @@ class EvictedClientCloser:
         self._clock = clock
         self._owned: weakref.WeakSet[object] = weakref.WeakSet()
         self._buckets: dict[_BucketKey, deque[_PendingClose]] = {}  # mutable-ok: deadline-ordered queues
+        self._dead_loop_tokens: deque[int] = deque()  # mutable-ok: loop finalizers append without taking the queue lock
+        self._watched_loop_tokens: set[int] = set()  # mutable-ok: tracks tokens until their dead buckets are discarded
         self._pending_count = 0
         self._queue_lock = threading.Lock()  # the cache is reachable from every worker thread's loop
         self._close_tasks: set[asyncio.Task[None]] = set()  # mutable-ok: strong refs to running closes
@@ -197,13 +214,19 @@ class EvictedClientCloser:
         close_fn: Final = _close_function(client)
         if close_fn is None:
             return
+        loop_token: Final = running_loop_token()
+        needs_loop: Final = inspect.iscoroutinefunction(close_fn)
         if self._pending_count >= self._max_pending:
-            return
+            self._discard_dead_loops()
+            if self._pending_count >= self._max_pending:
+                return
+        if needs_loop and loop_token is not None:
+            self._watch_loop(loop_token)
         self._enqueue(
             _PendingClose(
                 client_ref=weakref.ref(client),
-                loop_id=_running_loop_id(),
-                needs_loop=inspect.iscoroutinefunction(close_fn),
+                loop_token=loop_token,
+                needs_loop=needs_loop,
                 close_after=self._clock() + self._grace_seconds,
             )
         )
@@ -217,7 +240,7 @@ class EvictedClientCloser:
         if not self._pending_count:
             return
         now: Final = self._clock()
-        for pending in self._take_due(_running_loop_id(), now):
+        for pending in self._take_due(running_loop_token(), now):
             client = pending.client_ref()
             if client is None:
                 continue
@@ -230,6 +253,29 @@ class EvictedClientCloser:
     def pending_count(self) -> int:
         return self._pending_count
 
+    def _watch_loop(self, loop_token: int) -> None:
+        if loop_token in self._watched_loop_tokens:
+            return
+        try:
+            weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
+        except TypeError:
+            return
+        self._watched_loop_tokens.add(loop_token)
+
+    def _discard_dead_loops(self) -> None:
+        with self._queue_lock:
+            self._discard_dead_loops_locked()
+
+    def _discard_dead_loops_locked(self) -> None:
+        while self._dead_loop_tokens:
+            self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
+
+    def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
+        self._watched_loop_tokens.discard(loop_token)
+        bucket: Final = self._buckets.pop(loop_token, None)
+        if bucket is not None:
+            self._pending_count -= len(bucket)
+
     def _enqueue(self, pending: _PendingClose) -> None:
         """Append to the entry's bucket, dropping any dead entries it queues behind.
 
@@ -238,6 +284,7 @@ class EvictedClientCloser:
         the front rather than having to be searched for.
         """
         with self._queue_lock:
+            self._discard_dead_loops_locked()
             bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
             while bucket and bucket[0].client_ref() is None:
                 bucket.popleft()
@@ -245,9 +292,12 @@ class EvictedClientCloser:
             bucket.append(pending)
             self._pending_count += 1
 
-    def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
-        buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
+    def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
+        buckets = (
+            (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
+        )
         with self._queue_lock:
+            self._discard_dead_loops_locked()
             return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
 
     def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

litellm/caching/llm_caching_handler.py

@@ -2,10 +2,9 @@
 Add the event loop to the cache key, to prevent event loop closed errors.
 """
 
-import asyncio
 from typing import Final
 
-from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
+from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
 from .in_memory_cache import InMemoryCache
 
 
@@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
         self.evicted_client_closer.schedule(evicted)
         self.evicted_client_closer.reap()
 
-    def update_cache_key_with_event_loop(self, key):
+    def update_cache_key_with_event_loop(self, key: str) -> str:
         """
         Add the event loop to the cache key, to prevent event loop closed errors.
         If none, use the key as is.
         """
-        try:
-            event_loop: Final = asyncio.get_running_loop()
-            stringified_event_loop: Final = str(id(event_loop))
-            return f"{key}-{stringified_event_loop}"
-        except RuntimeError:  # handle no current running event loop
-            return key
+        token: Final = running_loop_token()
+        return f"{key}-{token}" if token is not None else key
 
-    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+    def _scoped_key(self, key: str, loop_bound: bool) -> str:
+        return self.update_cache_key_with_event_loop(key) if loop_bound else key
+
+    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
         """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
         if litellm_owned_client:
             self.evicted_client_closer.mark_owned(value)
-        key = self.update_cache_key_with_event_loop(key)
-        return super().set_cache(key, value, **kwargs)
+        return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
 
     async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
         if litellm_owned_client:
@@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
         key = self.update_cache_key_with_event_loop(key)
         return await super().async_set_cache(key, value, **kwargs)
 
-    def get_cache(self, key, **kwargs):
-        key = self.update_cache_key_with_event_loop(key)
+    def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
+        scoped_key: Final = self._scoped_key(key, loop_bound)
         self.evicted_client_closer.reap()
 
-        return super().get_cache(key, **kwargs)
+        return super().get_cache(scoped_key, **kwargs)
 
-    async def async_get_cache(self, key, **kwargs):
+    async def async_get_cache(self, key: str, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
 
         return await super().async_get_cache(key, **kwargs)

litellm/caching/redis_cache.py

@@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
     "return count"
 )
 
+_INCREMENT_IF_PRESENT_LUA: Final = (
+    "if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
+)
+
 _LUA_COUNT: Final = TypeAdapter(int)
+_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
 _OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
 
 
@@ -675,7 +680,7 @@ class RedisCache(BaseCache):
         # redis namespaces
         self.namespace = namespace
         # for high traffic, we store the redis results in memory and then batch write to redis
-        self.redis_batch_writing_buffer: list = []
+        self.redis_batch_writing_buffer: list[tuple[str, object]] = []
         if redis_flush_size is None:
             self.redis_flush_size: int = 100
         else:
@@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
         count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
         return _LUA_COUNT.validate_python(count)
 
+    @_redis_circuit_breaker_guard
+    async def async_increment_if_present(
+        self, key: str, value: float, parent_otel_span: Span | None = None
+    ) -> float | None:
+        _redis_client: Final = self._async_commands()
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        start_time: Final = time.time()
+        try:
+            total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
+                await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
+            )
+        except Exception as e:
+            failed_at: Final = time.time()
+            asyncio.create_task(
+                self.service_logger_obj.async_service_failure_hook(
+                    service=ServiceTypes.REDIS,
+                    duration=failed_at - start_time,
+                    error=e,
+                    call_type="async_increment_if_present",
+                    caller=_get_call_stack_info(),
+                    start_time=start_time,
+                    end_time=failed_at,
+                    parent_otel_span=parent_otel_span,
+                )
+            )
+            raise
+        end_time: Final = time.time()
+        asyncio.create_task(
+            self.service_logger_obj.async_service_success_hook(
+                service=ServiceTypes.REDIS,
+                duration=end_time - start_time,
+                call_type="async_increment_if_present",
+                caller=_get_call_stack_info(),
+                start_time=start_time,
+                end_time=end_time,
+                parent_otel_span=parent_otel_span,
+            )
+        )
+        return total
+
     async def flush_cache_buffer(self):
         print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
         await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
     return not _supports_audio_input(model, litellm_params)
 
 
-def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
+def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
     return MappingProxyType(
         {
             key: value
@@ -1066,11 +1066,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
         )
-        # OpenAI's Responses API rejects an empty input. For a system-only
-        # request, carry the system message as a system-role input item instead
-        # of instructions, mirroring how non-string system content is already
-        # handled in convert_chat_completion_messages_to_responses_api.
-        is_system_only_request: Final = not converted_input_items and converted_instructions is not None
+        # Keep the system text in input instead of instructions when the Responses API needs it there.
+        # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions.
+        typed_optional_params: Final = cast("Mapping[str, object]", optional_params)  # cast-ok: untyped dict
+        system_goes_in_input: Final = converted_instructions is not None and (
+            not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object"
+        )
         target_input_items: Final = (
             _without_audio_input_parts(converted_input_items)
             if _drops_audio_input(model, litellm_params)
@@ -1082,12 +1083,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     "type": "message",
                     "role": "system",
                     "content": [{"type": "input_text", "text": converted_instructions}],
-                }
+                },
+                *target_input_items,
             ]
-            if is_system_only_request
+            if system_goes_in_input
             else target_input_items
         )
-        instructions: Final = None if is_system_only_request else converted_instructions
+        instructions: Final = None if system_goes_in_input else converted_instructions
 
         # Build responses API request using the reverse transformation logic
         responses_api_request: Final = ResponsesAPIOptionalRequestParams()
@@ -1455,7 +1457,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
-        for key, value in _provider_metadata(raw_response.model_extra).items():
+        for key, value in responses_provider_metadata(raw_response.model_extra).items():
             setattr(model_response, key, value)
 
         # Preserve hidden params from the ResponsesAPIResponse, especially the headers
@@ -2139,7 +2141,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 from litellm.responses.utils import ResponseAPILoggingUtils
 
                 usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
-            provider_metadata: Final = _provider_metadata(response_data)
+            provider_metadata: Final = responses_provider_metadata(response_data)
             served_service_tier: Final = response_data.get("service_tier")
             return ModelResponseStream(
                 choices=[

litellm/constants.py

@@ -389,7 +389,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
 
-BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
+BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
 BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
 BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@@ -446,6 +446,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
 REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
 REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
 REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
+REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
 MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
 REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
 REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
@@ -701,6 +702,7 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv(
 )
 
 LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED: Final = 499
+LITELLM_CLIENT_DISCONNECT_DETAIL: Final = "Client disconnected the request"
 
 EMAIL_BUDGET_ALERT_TTL: Final = get_env_int("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)  # 24 hours in seconds
 EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = get_env_float(
@@ -1613,6 +1615,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
 MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
 MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
 PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
+INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
 
 # Headers to control callbacks
 X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
@@ -1624,8 +1627,10 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
 CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
+VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
+BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
 SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
@@ -1807,6 +1812,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
 
 ########################### DB CRON JOB NAMES ###########################
 DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
+TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
 DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
 PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
 CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"
@@ -1856,6 +1862,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
 PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
 PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
 PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
+TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
 RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))
@@ -1950,6 +1957,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "public_model_groups",
     "public_model_groups_links",
     "cost_discount_config",
+    "apply_search_free_tier",
     "cost_margin_config",
     "block_requests_for_models_without_pricing",
     "budget_exceeded_throttle_percentage",
@@ -1980,6 +1988,7 @@ SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30
 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000
 SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000
+SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS: Final = 30
 SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE: Final = 100
 # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
 # callers from forcing a DB query per request for unknown names, while bounding

litellm/cost_calculator.py

@@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
 from httpx import Response
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@@ -42,9 +42,12 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     generic_cost_per_token,
     get_batch_cost_rates,
     get_billable_input_tokens,
+    get_cost_per_unit,
+    get_modality_token_cost_key,
     get_regional_uplift_multiplier,
     get_service_tier_cost_key,
     get_token_type_cost_breakdown,
+    get_vertex_regional_endpoint_uplift,
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
@@ -134,6 +137,7 @@ from litellm.types.utils import (
     TranscriptionUsageTokensObject,
     Usage,
     VectorStoreSearchResponse,
+    is_server_derived_pricing_key,
 )
 from litellm.utils import (
     CallTypes,
@@ -146,7 +150,10 @@ from litellm.utils import (
     TextCompletionResponse,
     TranscriptionResponse,
     cached_get_model_info_helper,
+    model_info_with_pricing_overrides,
+    runtime_model_cost_registrations,
     token_counter,
+    unmapped_model_generalization,
 )
 
 if TYPE_CHECKING:
@@ -424,6 +431,7 @@ def cost_per_token(
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
     custom_model_info: OCRPricing | None = None,
+    bedrock_model_info: ModelInfo | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -663,6 +671,7 @@ def cost_per_token(
             model=model,
             custom_llm_provider=custom_llm_provider,
             data_residency=data_residency,
+            vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
         if _transcription_usage_has_token_details(usage_block):
@@ -725,6 +734,15 @@ def cost_per_token(
     elif custom_llm_provider == "anthropic":
         return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "bedrock":
+        resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False)
+        if resolved_bedrock_info is not None:
+            return generic_cost_per_token(
+                model=model,
+                usage=usage_block,
+                custom_llm_provider="bedrock",
+                service_tier=service_tier,
+                model_info=resolved_bedrock_info,
+            )
         return bedrock_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "openai":
         return openai_cost_per_token(
@@ -847,6 +865,37 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
     )
 
 
+def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool:
+    return _cost_map_entry_prices_anything(entry) or any(
+        value is not None and field.startswith("cache_") and is_server_derived_pricing_key(field)
+        for field, value in entry.items()
+    )
+
+
+def _bedrock_pricing_info(
+    model: str | None,
+    invoked_model_id: str | None,
+    custom_llm_provider: str | None,
+    explicit_pricing: bool,
+) -> ModelInfo | None:
+    if explicit_pricing or custom_llm_provider != "bedrock" or not model:
+        return None
+    registration: Final = next(
+        (
+            (key, entry)
+            for key, entry in runtime_model_cost_registrations(model, "bedrock")
+            if _bedrock_registration_has_prices(entry)
+        ),
+        None,
+    )
+    profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
+    if profile is None or not _declares_rates(profile):
+        return _cost_map_model_info(registration[0], None) if registration is not None else None
+    if registration is None or registration[0] == profile["key"]:
+        return profile
+    return model_info_with_pricing_overrides(profile, registration[1])
+
+
 def _declares_rates(model_info: ModelInfo | None) -> bool:
     entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
     return entry is not None and _cost_map_entry_prices_anything(entry)
@@ -863,6 +912,26 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
     return own_model_info["litellm_provider"]
 
 
+def _pricing_provider_for_cost_candidate(
+    candidate_provider: str | None,
+    request_provider: str | None,
+    current_provider: object,
+) -> str | None:
+    if candidate_provider != request_provider:
+        return candidate_provider
+    return current_provider if isinstance(current_provider, str) else None
+
+
+def _invoked_model_id_for_cost(
+    optional_params: object,
+    logging_obj: LitellmLoggingObject | None,
+) -> str | None:
+    recorded: Final[object] = getattr(logging_obj, "model_call_details", None)
+    return _get_hidden_str_for_cost_calc(optional_params, "model_id") or _get_hidden_str_for_cost_calc(
+        recorded, "model_id"
+    )
+
+
 def select_model_name_for_cost_calc(
     model: str | None,
     completion_response: object | None,
@@ -871,6 +940,7 @@ def select_model_name_for_cost_calc(
     custom_llm_provider: str | None = None,
     router_model_id: str | None = None,
     region_name: str | None = None,
+    invoked_model_id: str | None = None,
 ) -> str | None:
     """
     1. If custom pricing is true, return received model name
@@ -891,6 +961,13 @@ def select_model_name_for_cost_calc(
     hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
+    configured_model: Final = model or (
+        completion_response_model if isinstance(completion_response_model, str) else None
+    )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        configured_model, invoked_model_id, custom_llm_provider, explicit_pricing
+    )
+    bedrock_pricing_model: Final = bedrock_model_info["key"] if bedrock_model_info is not None else None
     priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
     priced_region: Final = (
         _get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name
@@ -908,8 +985,8 @@ def select_model_name_for_cost_calc(
         else:
             return_model = model
 
-    elif base_model is not None or provider_response_model is not None:
-        return_model = base_model if base_model is not None else provider_response_model
+    elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
+        return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
 
     elif completion_response_model is None and hidden_params is not None:
         if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
@@ -1538,6 +1615,10 @@ def completion_cost(
             if base_model is not None and custom_pricing is not True
             else None
         )
+        invoked_model_id: Final = _invoked_model_id_for_cost(
+            _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None,
+            litellm_logging_obj,
+        )
         selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
@@ -1546,17 +1627,30 @@ def completion_cost(
             base_model=base_model,
             router_model_id=router_model_id,
             region_name=region_name,
+            invoked_model_id=invoked_model_id,
         )
 
+        bedrock_model_info: Final = _bedrock_pricing_info(
+            model or _get_response_model(completion_response),
+            invoked_model_id,
+            _get_provider_for_cost_calc(model, custom_llm_provider),
+            explicit_pricing,
+        )
         requested_model: Final = model
-        potential_model_names: Final = [
-            selected_model,
-            _get_response_model(completion_response),
-        ]
-        if model is not None:
-            potential_model_names.append(model)
+        request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
+        raw_candidates: Final = (
+            (
+                selected_model,
+                base_model_provider if base_model_provider is not None else request_pricing_provider,
+            ),
+            (_get_response_model(completion_response), request_pricing_provider),
+            *(((model, request_pricing_provider),) if model is not None else ()),
+        )
+        potential_model_names: Final = (
+            raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
+        )
 
-        for idx, model in enumerate(potential_model_names):
+        for idx, (model, pricing_provider) in enumerate(potential_model_names):
             try:
                 if verbose_logger.isEnabledFor(logging.DEBUG):
                     verbose_logger.debug("selected model name for cost calculation: %s", model)
@@ -1895,13 +1989,6 @@ def completion_cost(
                 if litellm_logging_obj is not None:
                     request_model_for_cost = litellm_logging_obj.model
 
-                pricing_provider: str | None = (
-                    base_model_provider
-                    if base_model_provider is not None and idx == 0
-                    else custom_llm_provider
-                    if isinstance(custom_llm_provider, str)
-                    else None
-                )
                 (
                     prompt_tokens_cost_usd_dollar,
                     completion_tokens_cost_usd_dollar,
@@ -1909,7 +1996,11 @@ def completion_cost(
                     model=model,
                     prompt_tokens=prompt_tokens or 0,
                     completion_tokens=completion_tokens or 0,
-                    custom_llm_provider=pricing_provider,
+                    custom_llm_provider=_pricing_provider_for_cost_candidate(
+                        candidate_provider=pricing_provider,
+                        request_provider=request_pricing_provider,
+                        current_provider=custom_llm_provider,
+                    ),
                     response_time_ms=response_time_ms,
                     region_name=None if explicit_pricing else region_name,
                     custom_cost_per_second=custom_cost_per_second,
@@ -1927,6 +2018,7 @@ def completion_cost(
                     vertex_location=vertex_location,
                     response=completion_response,
                     custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
+                    bedrock_model_info=bedrock_model_info,
                 )
 
                 # Get additional costs from provider (e.g., routing fees, infrastructure costs)
@@ -2008,12 +2100,17 @@ def completion_cost(
                     if cost_per_token_usage_object is not None and model:
                         _token_type_breakdown = get_token_type_cost_breakdown(
                             model=model,
-                            custom_llm_provider=pricing_provider,
+                            custom_llm_provider=_pricing_provider_for_cost_candidate(
+                                candidate_provider=pricing_provider,
+                                request_provider=request_pricing_provider,
+                                current_provider=custom_llm_provider,
+                            ),
                             usage=cost_per_token_usage_object,
                             service_tier=service_tier,
                             data_residency=data_residency,
                             vertex_location=vertex_location,
                             custom_cost_per_token=custom_cost_per_token,
+                            model_info=bedrock_model_info,
                         )
                         _reasoning_cost = _token_type_breakdown.reasoning_cost
                         _cache_read_cost = _token_type_breakdown.cache_read_cost
@@ -2263,6 +2360,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
         return None
 
 
+_CostCandidate: TypeAlias = tuple[str | None, str | None]
+
+
+def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
+    model, pricing_provider = candidate
+    if model is None:
+        return False
+    rule_info: Final = unmapped_model_generalization(model, pricing_provider)
+    return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
+
+
+def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
+    unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
+    return (
+        *(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
+        *(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
+    )
+
+
 def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
     raw_entry: Final = litellm.model_cost.get(key)
     return raw_entry if isinstance(raw_entry, Mapping) else None
@@ -2290,7 +2406,16 @@ def pricing_entry_for_cost_calc(
         custom_llm_provider=custom_llm_provider,
         router_model_id=router_model_id,
         region_name=region_name,
+        invoked_model_id=_invoked_model_id_for_cost(None, litellm_logging_obj),
     )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        model or _get_response_model(completion_response),
+        _invoked_model_id_for_cost(None, litellm_logging_obj),
+        _get_provider_for_cost_calc(model, custom_llm_provider),
+        custom_pricing is True or base_model is not None,
+    )
+    if bedrock_model_info is not None:
+        return bedrock_model_info["key"], bedrock_model_info
     candidates: Final = (selected_model, _get_response_model(completion_response), model)
     resolved: Final = next(
         (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info),
@@ -2709,12 +2834,17 @@ def _batch_rate(
     model_info: ModelInfo,
     key: Literal[
         "input_cost_per_audio_token_batches",
+        "cache_read_input_audio_token_cost_batches",
         "input_cost_per_image_token_batches",
         "input_cost_per_video_token_batches",
     ],
     fallback: float,
+    *,
+    prompt_tokens: int = 0,
 ) -> float:
-    rate: Final = model_info.get(key)
+    context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
+    context_rate: Final = get_cost_per_unit(model_info, context_key, None)
+    rate: Final = model_info.get(key) if context_rate is None else context_rate
     return fallback if rate is None else rate
 
 
@@ -2738,6 +2868,8 @@ def batch_cost_calculator(
     custom_llm_provider: str | None = None,
     model_info: ModelInfo | None = None,
     data_residency: str | None = None,
+    *,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """
     Calculate the cost of a batch job.
@@ -2827,7 +2959,8 @@ def batch_cost_calculator(
         total_prompt_cost *= uplift
         total_completion_cost *= uplift
 
-    return total_prompt_cost, total_completion_cost
+    vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
+    return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
 
 
 def _batch_prompt_cost(
@@ -2839,6 +2972,7 @@ def _batch_prompt_cost(
 ) -> float:
     details: Final = parse_prompt_tokens_details(usage)
     cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
+    cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
     written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
     audio_tokens, image_tokens, video_tokens = (
         details["audio_tokens"],
@@ -2850,10 +2984,13 @@ def _batch_prompt_cost(
     )
     return (
         text_tokens * input_rate
-        + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+        + audio_tokens
+        * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
         + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
         + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
-        + cached_tokens * (cache_read_rate or 0.0)
+        + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+        + cached_audio_tokens
+        * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
         + written_tokens * (cache_creation_rate or 0.0)
     )
 

litellm/decisions/call.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
@@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias
 
 import httpx
 from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
 from typing_extensions import assert_never
 
 import litellm
@@ -56,6 +58,7 @@ from litellm.types.decisions import (
     OpenAIDecisionResponse,
     UnsupportedDecisionsRequest,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
 from litellm.utils import ProviderConfigManager
 
@@ -415,6 +418,30 @@ def _outbound_headers(
     return MappingProxyType({**(extra_headers or {}), **forwarded})
 
 
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64")
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
 def prepare_call(
     *,
     model: str,

litellm/decisions/main.py

@@ -15,6 +15,7 @@ from litellm.decisions.call import (
     SystemOneQuestions,
     SystemOneRequestFields,
     asend,
+    log_caller_request,
     prepare_call,
     reject_images_next_to_input,
     send,
@@ -76,6 +77,16 @@ def _prepare_decisions_call(
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
 ) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
+        {
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
+    )
     return prepare_call(
         model=model,
         fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),

litellm/exceptions.py

@@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
         )
 
 
+class MalformedRequestFieldError(BadRequestError):
+    pass
+
+
 class UnprocessableEntityError(openai.UnprocessableEntityError):
     def __init__(
         self,

litellm/images/main.py

@@ -29,7 +29,6 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.custom_llm import CustomLLM
 from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
 
 #################### Initialize provider clients ####################
@@ -523,10 +522,7 @@ def image_generation(
             )
         elif custom_llm_provider in litellm._custom_providers:  # Assume custom LLM provider
             # Get the Custom Handler
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -799,10 +795,7 @@ def image_edit(
 
         # Check for custom provider
         if custom_llm_provider in litellm._custom_providers:
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)

litellm/integrations/agentops/agentops.py

@@ -99,18 +99,14 @@ class AgentOps(OpenTelemetry):
             "Connection": "keep-alive",
         }
 
-        client: Final = get_httpx_client()
-        try:
-            response: Final = client.post(
-                url=auth_endpoint,
-                headers=headers,
-                json={"api_key": api_key},
-                timeout=10,
-            )
+        response: Final = get_httpx_client().post(
+            url=auth_endpoint,
+            headers=headers,
+            json={"api_key": api_key},
+            timeout=10,
+        )
 
-            if response.status_code != 200:
-                raise Exception(f"Failed to fetch auth token: {response.text}")
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch auth token: {response.text}")
 
-            return response.json()
-        finally:
-            client.close()
+        return response.json()

litellm/integrations/custom_guardrail.py

@@ -3,7 +3,7 @@ import copy
 import hashlib
 import os
 import secrets
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
@@ -69,6 +69,8 @@ GUARDRAIL_SESSIONS_TARGET: Final = "guardrail_sessions"
 # field to suppress a guardrail on the direct-SDK path that never reaches the
 # proxy's metadata sanitizer.
 _PRE_CALL_EXECUTED_TOKEN: Final = secrets.token_hex(16)
+_PRE_CALL_RAISED_TOKEN: Final = secrets.token_hex(16)
+_DEPLOYMENT_HOOK_RAN_TOKEN: Final = secrets.token_hex(16)
 
 _GUARDRAIL_BLOCK_STATUS_CODES: Final = frozenset({400, 403, 422})
 
@@ -218,6 +220,25 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
     )
 
 
+def _str_collection(value: object) -> frozenset[str]:
+    if not isinstance(value, (list, tuple, set, frozenset)):
+        return frozenset()
+    items: Final = cast("Iterable[object]", value)  # cast-ok: a metadata collection of guardrail names
+    return frozenset(item for item in items if isinstance(item, str))
+
+
+def _request_carries_marker(data: Mapping[str, object], marker: str | None) -> bool:
+    if marker is None:
+        return False
+    for meta_key in ("metadata", "litellm_metadata"):
+        meta = data.get(meta_key)
+        if isinstance(meta, dict):
+            executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
+            if isinstance(executed, list) and marker in executed:
+                return True
+    return False
+
+
 class CustomGuardrail(CustomLogger):
     # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
     use_native_during_call_hook: ClassVar[bool] = False
@@ -824,13 +845,16 @@ class CustomGuardrail(CustomLogger):
 
         return False
 
-    def _pre_call_marker(self) -> str | None:
+    def _marker(self, token: str) -> str | None:
         name: Final = self.guardrail_name
         if not name:
             return None
-        return f"{_PRE_CALL_EXECUTED_TOKEN}:{name}"
+        return f"{token}:{name}"
 
-    def mark_pre_call_hook_ran(self, data: dict[str, object]) -> None:
+    def _pre_call_marker(self, *, raised: bool = False) -> str | None:
+        return self._marker(_PRE_CALL_RAISED_TOKEN if raised else _PRE_CALL_EXECUTED_TOKEN)
+
+    def mark_pre_call_hook_ran(self, data: dict[str, object], *, raised: bool = False) -> None:
         """
         Record that this guardrail's ``async_pre_call_hook`` already ran for this
         request, so the deployment-level hook does not run it a second time.
@@ -839,8 +863,11 @@ class CustomGuardrail(CustomLogger):
         router later spreads a deployment's model-level ``guardrails`` into the
         top-level request kwargs, which would otherwise re-trigger the same hook
         from ``async_pre_call_deployment_hook``.
+
+        With ``raised``, records instead that the hook raised as a policy pipeline
+        step the pipeline then moved past. The deployment-level hook still runs it.
         """
-        marker: Final = self._pre_call_marker()
+        marker: Final = self._pre_call_marker(raised=raised)
         if marker is None:
             return
         for meta_key in ("metadata", "litellm_metadata"):
@@ -855,17 +882,57 @@ class CustomGuardrail(CustomLogger):
                 return
         data["metadata"] = {PRE_CALL_EXECUTED_GUARDRAILS_KEY: [marker]}
 
-    def _pre_call_hook_already_ran(self, data: dict[str, object]) -> bool:
-        marker: Final = self._pre_call_marker()
-        if marker is None:
-            return False
-        for meta_key in ("metadata", "litellm_metadata"):
-            meta = data.get(meta_key)
-            if isinstance(meta, dict):
-                executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
-                if isinstance(executed, list) and marker in executed:
-                    return True
-        return False
+    def pre_call_hook_ran_or_raised(self, data: Mapping[str, object]) -> bool:
+        """Whether this guardrail's ``async_pre_call_hook`` ran for this request, whatever it decided."""
+        return self._pre_call_hook_already_ran(data) or self._pre_call_hook_already_ran(data, raised=True)
+
+    def _pre_call_hook_already_ran(self, data: Mapping[str, object], *, raised: bool = False) -> bool:
+        return _request_carries_marker(data, self._pre_call_marker(raised=raised))
+
+    def deployment_hook_ran(self, data: Mapping[str, object]) -> bool:
+        """Whether a deployment-level hook ran this guardrail as a policy pipeline step of this request."""
+        return _request_carries_marker(data, self._marker(_DEPLOYMENT_HOOK_RAN_TOKEN))
+
+    def _record_deployment_hook_run(
+        self,
+        data: dict[str, object],  # mutable-ok: the request payload the run is recorded on
+    ) -> None:
+        """
+        Record that a deployment-level hook ran this guardrail when it is a policy pipeline step of
+        this request, and name it in the applied guardrails when the request's metadata lists it.
+        """
+        name: Final = self.guardrail_name
+        marker: Final = self._marker(_DEPLOYMENT_HOOK_RAN_TOKEN)
+        buckets: Final = tuple(
+            cast("dict[str, object]", bucket)  # cast-ok: metadata buckets are str-keyed
+            for bucket in (data.get("metadata"), data.get("litellm_metadata"))
+            if isinstance(bucket, dict)
+        )
+        policy_state: Final = next(
+            (bucket for bucket in buckets if name in _str_collection(bucket.get("_pipeline_managed_guardrails"))),
+            None,
+        )
+        if marker is None or policy_state is None:
+            return
+        recorded: Final = policy_state.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
+        if not isinstance(recorded, list):
+            policy_state[PRE_CALL_EXECUTED_GUARDRAILS_KEY] = [marker]
+        elif marker not in recorded:
+            cast("list[object]", recorded).append(marker)  # cast-ok: markers are only compared as strings
+        if not any(name in _str_collection(bucket.get("guardrails")) for bucket in buckets):
+            return
+        named_lists: Final = tuple(
+            cast("list[object]", bucket["applied_guardrails"])  # cast-ok: entries are only compared with names
+            for bucket in buckets
+            if isinstance(bucket.get("applied_guardrails"), list)
+        )
+        for applied in named_lists:
+            if name not in applied:
+                applied.append(name)
+        if named_lists:
+            return
+        metadata_bucket: Final = cast("dict[str, object]", get_or_create_metadata_bucket(data)[1])  # cast-ok: str keys
+        metadata_bucket["applied_guardrails"] = [name]
 
     @classmethod
     def uses_apply_guardrail_interface(cls) -> bool:
@@ -914,12 +981,15 @@ class CustomGuardrail(CustomLogger):
             target: Final = self._deployment_hook_target()
             if target is not self:
                 kwargs["guardrail_to_apply"] = self
-            result: Final = await target.async_pre_call_hook(
-                user_api_key_dict=_user_api_key_auth_from_request(kwargs),
-                cache=dc,
-                data=kwargs,
-                call_type="completion" if call_type == CallTypes.completion else "acompletion",
-            )
+            try:
+                result: Final = await target.async_pre_call_hook(
+                    user_api_key_dict=_user_api_key_auth_from_request(kwargs),
+                    cache=dc,
+                    data=kwargs,
+                    call_type="completion" if call_type == CallTypes.completion else "acompletion",
+                )
+            finally:
+                self._record_deployment_hook_run(kwargs)
 
             if result is not None and isinstance(result, dict):
                 result_messages: Final = result.get("messages")
@@ -1003,6 +1073,9 @@ class CustomGuardrail(CustomLogger):
         finally:
             if target is not self:
                 request_data.pop("guardrail_to_apply", None)
+            self._record_deployment_hook_run(
+                cast("dict[str, object]", request_data)  # cast-ok: proxy request payloads are str-keyed
+            )
 
         if not self._is_valid_response_type(result):
             return None
@@ -1139,6 +1212,17 @@ class CustomGuardrail(CustomLogger):
         """
         return False
 
+    def supports_structured_messages(self) -> bool:
+        """Whether this guardrail scans every row of ``structured_messages`` and returns texts that agree with them.
+
+        Guardrails that scan only part of the rows (the latest user turn, the
+        last row, the turns after the last assistant turn, or no tool results),
+        derive their returned texts from the rows, or cannot rewrite rows whose
+        content is a list of parts, override this to return False. Translation handlers that can rewrite the request from ``texts``
+        alone, like Bedrock Converse, then hand them only ``texts``.
+        """
+        return True
+
     def _suppressed_by_auto_router_compression(self) -> bool:
         """True when an auto router's own compression policy suppresses this guardrail.
 

litellm/integrations/custom_logger.py

@@ -898,6 +898,11 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
     def redacts_messages_itself(self) -> bool:
         return False
 
+    def exempt_from_message_redaction(self) -> bool:
+        """Router pre-call checks that key routing off the request and never log it return True, so
+        `turn_off_message_logging` leaves the success event they receive unredacted."""
+        return False
+
     def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict:
         """
         Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.

litellm/integrations/gcs_bucket/gcs_bucket_base.py

@@ -20,7 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
     httpxSpecialProvider,
 )
 from litellm.types.integrations.gcs_bucket import *
-from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
+from litellm.types.utils import StandardLoggingPayload
 
 if TYPE_CHECKING:
     from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -140,49 +140,16 @@ class GCSBucketBase(CustomBatchLogger):
 
     async def get_gcs_logging_config(self, kwargs: dict[str, Any] | None = {}) -> GCSLoggingConfig:
         """
-        This function is used to get the GCS logging config for the GCS Bucket Logger.
-        It checks if the dynamic parameters are provided in the kwargs and uses them to get the GCS logging config.
-        If no dynamic parameters are provided, it uses the default values.
+        Returns the bucket and service account this logger was configured with.
         """
-        if kwargs is None:
-            kwargs = {}
-
-        standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
-            "standard_callback_dynamic_params", None
-        )
-
-        bucket_name: str
-        path_service_account: str | None
-        if standard_callback_dynamic_params is not None:
-            verbose_logger.debug("Using dynamic GCS logging")
-            verbose_logger.debug("standard_callback_dynamic_params: %s", standard_callback_dynamic_params)
-
-            _bucket_name: str | None = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME
-            _path_service_account: Final[str | None] = (
-                standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json
+        if self.BUCKET_NAME is None:
+            raise ValueError(
+                "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
             )
-
-            if _bucket_name is None:
-                raise ValueError(
-                    "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
-                )
-            bucket_name = _bucket_name
-            path_service_account = _path_service_account
-            vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
-        else:
-            # If no dynamic parameters, use the default instance
-            if self.BUCKET_NAME is None:
-                raise ValueError(
-                    "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
-                )
-            bucket_name = self.BUCKET_NAME
-            path_service_account = self.path_service_account_json
-            vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
-
         return GCSLoggingConfig(
-            bucket_name=bucket_name,
-            vertex_instance=vertex_instance,
-            path_service_account=path_service_account,
+            bucket_name=self.BUCKET_NAME,
+            vertex_instance=await self.get_or_create_vertex_instance(credentials=self.path_service_account_json),
+            path_service_account=self.path_service_account_json,
         )
 
     async def get_or_create_vertex_instance(self, credentials: str | None) -> VertexBase:

litellm/integrations/otel/README.md

@@ -91,7 +91,7 @@ way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue
 rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
 module under `litellm/` and `enterprise/` that calls a shared cache or declares a
 read or write on the request batch (`reserve_redis_batch_reads`,
-`declare_batch_get`, `batch.mget`, `batch.set`, `batch.script`) and fails when
+`batch.mget`, `batch.set`, `batch.script`) and fails when
 one has no declared family, with the process-local `InMemoryCache` callers listed
 as the only exemptions. A batch op carries the family that was active when it was
 declared, so the routing prefetch armed before deployment selection

litellm/integrations/otel/model/spans.py

@@ -214,6 +214,7 @@ _SERVICE_VERB_BY_CALL_TYPE: Final[dict[str, str]] = {
     "increment_cache": "incr",
     "async_increment": "incr",
     "async_increment_pipeline": "incr",
+    "async_increment_if_present": "incr",
     "delete_cache": "delete",
     "async_delete_cache": "delete",
     "async_rpush": "rpush",

litellm/integrations/otel/mount.py

@@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
 
     Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
     is unavailable. This MUST be called at app-creation time — once the lifespan
-    runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
-    middleware after an application has started".
+    runs, the middleware stack is frozen and middleware added later never runs, so
+    a late call is skipped without registering anything.
 
     No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
     ``ProxyTracerProvider``; the proxy publishes the real provider as the global
@@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
     try:
         if not is_otel_v2_enabled():
             return
+        if app.middleware_stack is not None:
+            verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
+            return
 
         # Lazy: only the V2-enabled path needs the optional
         # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

litellm/integrations/otel/plumbing/otlp_json.py

@@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
 
 import base64
 import json
+import logging
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
 from typing import Final, TypeAlias
@@ -13,9 +14,13 @@ from typing import Final, TypeAlias
 import requests
 from google.protobuf.json_format import MessageToDict
 from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
+from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
 
+_logger: Final = logging.getLogger(__name__)
 JSON_CONTENT_TYPE: Final = "application/json"
 _HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
 
@@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
     def __init__(
         self,
         endpoint: str | None,
-        headers: dict[str, str],  # mutable-ok: SDK __init__ takes Dict
+        headers: Mapping[str, str],
         certificate_file: str | None = None,
         session: "requests.Session | None" = None,
+        *,
+        max_request_size: int | None = None,
     ) -> None:
-        super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
-        self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
+        super().__init__(
+            endpoint=endpoint,
+            headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
+            certificate_file=certificate_file,
+            session=session,
+            max_request_size=max_request_size,
+        )
 
-    def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
-        return encode_spans_json(spans)
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        if self._shutdown:
+            _logger.warning("Exporter already shutdown, ignoring batch")
+            return SpanExportResult.FAILURE
+
+        with self._metrics.export_operation(len(spans)) as result:
+            try:
+                serialized_data: Final = encode_spans_json(spans)
+            except Exception as error:  # noqa: BLE001  # SDK export contract reports encoding failures as results
+                _logger.error("Failed to encode span batch: %s", error)
+                result.error = error
+                return SpanExportResult.FAILURE
+
+            if 0 < self._max_request_size < len(serialized_data):
+                result.error = RequestPayloadTooLargeError(
+                    f"Serialized span request size {len(serialized_data)} bytes "
+                    f"exceeds max_request_size {self._max_request_size} bytes."
+                )
+                _logger.warning("Dropping span batch: %s", result.error)
+                return SpanExportResult.FAILURE
+
+            export_result: Final = self._client.export(serialized_data)
+            if not export_result.success:
+                result.error = export_result.error
+                result.error_attrs = (
+                    {HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
+                    if export_result.status_code is not None
+                    else None
+                )
+                return SpanExportResult.FAILURE
+
+        return SpanExportResult.SUCCESS

litellm/integrations/otel/presets/agentops.py

@@ -76,6 +76,7 @@ def agentops_preset(
 
 def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
     """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
+    import requests
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
@@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
         """
 
         def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
-            super().__init__(endpoint=endpoint)
+            self._agentops_session = requests.Session()
+            super().__init__(endpoint=endpoint, session=self._agentops_session)
             self._agentops_api_key = api_key
             self._auth_resolved = False
 
@@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
             try:
                 token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
                 if token:
-                    # ``_session`` is the requests.Session the base exporter
-                    # POSTs through; updating its Authorization header is how the
-                    # minted JWT reaches every subsequent export.
-                    self._session.headers["Authorization"] = f"Bearer {token}"
+                    self._agentops_session.headers["Authorization"] = f"Bearer {token}"
             except Exception as e:
                 verbose_logger.debug("AgentOps JWT fetch failed: %s", e)
 

litellm/integrations/prometheus.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     PROXY_REJECTED_BEFORE_ROUTING_KEY,
 )
 from litellm.exceptions import (
+    MalformedRequestFieldError,
     validate_rate_limit_category,
     validate_rate_limit_type,
 )
@@ -258,6 +259,15 @@ _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field
 _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)])
 _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
 _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
+_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _logged_fields(value: object) -> Mapping[str, object]:
+    try:
+        return _LOGGED_FIELDS.validate_python(value)
+    except ValidationError:
+        return _NO_LOGGED_FIELDS
 
 
 def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@@ -2846,7 +2856,7 @@ class PrometheusLogger(CustomLogger):
 
     async def async_post_call_failure_hook(
         self,
-        request_data: dict,
+        request_data: dict[str, object],
         original_exception: Exception,
         user_api_key_dict: UserAPIKeyAuth,
         traceback_str: str | None = None,
@@ -2869,10 +2879,14 @@ class PrometheusLogger(CustomLogger):
         try:
             _tags: Final = StandardLoggingPayloadSetup.get_request_tags(
                 litellm_params=request_data,
-                proxy_server_request=request_data.get("proxy_server_request", {}),
+                proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
+            )
+            _metadata: Final = _logged_fields(request_data.get("metadata"))
+            model_id: Final = (
+                _logged_fields(_metadata.get("model_info")).get("id")
+                or _logged_fields(request_data.get("model_info")).get("id")
+                or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
             )
-            _metadata: Final = request_data.get("metadata", {}) or {}
-            model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
             rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
             api_provider: Final = self._extract_api_provider_from_request_data(
                 request_data
@@ -3104,7 +3118,11 @@ class PrometheusLogger(CustomLogger):
 
             # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
             # into requested_model and leave deployment-scoped labels empty.
-            deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+            deployment_selected: Final = (
+                bool(model_id)
+                and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+                and not isinstance(exception, MalformedRequestFieldError)
+            )
             if deployment_selected:
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id

litellm/litellm_core_utils/core_helpers.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c
 import httpx
 from pydantic import TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
@@ -92,6 +93,10 @@ def normalize_drop_params(value: object) -> bool | None:
         return None
 
 
+def drops_unsupported_params(drop_params: object) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(drop_params) is True
+
+
 def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool:
     normalized: Final = normalize_drop_params(value)
     if normalized is None and value is not None:
@@ -318,7 +323,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
             items.pop(index)
 
 
-def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
+def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
     """
     Helper to get litellm metadata for spend tracking
 
@@ -326,7 +331,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
     and user_api_key values are in 'metadata'.
     """
     potential_spend_tracking_metadata_substring: Final = "user_api_key"
-    for key, value in metadata.items():
+    for key, value in dict(metadata).items():
         if potential_spend_tracking_metadata_substring in key:
             litellm_metadata[key] = value
     return litellm_metadata

litellm/litellm_core_utils/get_model_cost_map.py

@@ -490,22 +490,42 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
     return _cost_map_source_info.loaded_at
 
 
-def _expand_model_aliases(model_cost: dict) -> dict:
+class AliasedCostEntry(dict[str, object]):
+    __slots__ = ("names", "registered_fields")
+
+    def __init__(
+        self, entry: Mapping[str, object], names: tuple[str, ...], registered_fields: frozenset[str] = frozenset()
+    ) -> None:
+        super().__init__(entry)
+        self.names: tuple[str, ...] = names
+        self.registered_fields: frozenset[str] = registered_fields
+
+
+def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
+    return MappingProxyType(
+        {
+            owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
+            for owner in dict.fromkeys(alias_owners.values())
+        }
+    )
+
+
+def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
     """
     Expand ``aliases`` lists in model cost entries into top-level entries.
 
-    Each alias gets a reference to the **same** dict object as the canonical
-    entry (zero memory overhead).  The ``aliases`` key is removed from the
-    entry so downstream code never sees it.
+    Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
+    canonical entry, which carries every name it is served under.  The
+    ``aliases`` key is removed from the entry so downstream code never sees it.
 
     If an alias collides with an existing canonical entry the alias is
     skipped and a warning is logged.
     """
-    aliases_to_add: Final[dict[str, dict]] = {}
+    alias_owners: Final[dict[str, str]] = {}
     keys_with_aliases: Final[list[str]] = []
 
     for model_name, model_info in model_cost.items():
-        aliases: list | None = model_info.get("aliases")
+        aliases: object = model_info.get("aliases")
         if aliases is None:
             continue
         keys_with_aliases.append(model_name)
@@ -527,7 +547,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
                     model_name,
                 )
                 continue
-            if alias in aliases_to_add:
+            if alias in alias_owners:
                 verbose_logger.warning(
                     "LiteLLM model alias conflict: alias '%s' (from '%s') "
                     "was already claimed by another entry — skipping.",
@@ -535,13 +555,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
                     model_name,
                 )
                 continue
-            aliases_to_add[alias] = model_info  # same dict reference
+            alias_owners[alias] = model_name
 
     # Remove the ``aliases`` key from entries so it doesn't pollute model info
     for key in keys_with_aliases:
         model_cost[key].pop("aliases", None)
 
-    model_cost.update(aliases_to_add)
+    for owner, names in _alias_groups(alias_owners).items():
+        model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
     return model_cost