Merge pull request #49802 from BerriAI/litellm_backport_49785_stable_1_106_x

feat(videos): serve custom_provider_map BaseVideoConfig providers on /v1/videos (#49785) [backport stable/1.106.x]

.circleci/config.yml

@@ -3378,38 +3378,14 @@ workflows:
         - not:
             equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
-      - using_litellm_on_windows:
-          filters:
-            branches:
-              only: main
-      - windows_release_wheel:
-          filters:
-            branches:
-              only: main
-      - provider_replay_harness:
-          filters:
-            branches:
-              only: main
-      - base_sdk_install:
-          filters:
-            branches:
-              only: main
-      - local_testing_part1:
-          filters:
-            branches:
-              only: main
-      - local_testing_part2:
-          filters:
-            branches:
-              only: main
-      - langfuse_logging_unit_tests:
-          filters:
-            branches:
-              only: main
-      - litellm_router_testing:
-          filters:
-            branches:
-              only: main
+      - using_litellm_on_windows
+      - windows_release_wheel
+      - provider_replay_harness
+      - base_sdk_install
+      - local_testing_part1
+      - local_testing_part2
+      - langfuse_logging_unit_tests
+      - litellm_router_testing
       - auth_ui_unit_tests
       - build_docker_database_image
       - e2e_ui_testing
@@ -3429,65 +3405,26 @@ workflows:
       - proxy_store_model_in_db_tests:
           requires:
             - build_docker_database_image
-      - proxy_build_from_pip_tests:
-          filters:
-            branches:
-              only: main
+      - proxy_build_from_pip_tests
       - proxy_pass_through_endpoint_tests:
           requires:
             - build_docker_database_image
       - proxy_e2e_anthropic_messages_tests:
           requires:
             - build_docker_database_image
-      - llm_translation_testing:
-          filters:
-            branches:
-              only: main
-      - realtime_translation_testing:
-          filters:
-            branches:
-              only: main
+      - llm_translation_testing
+      - realtime_translation_testing
       - guardrails_testing
-      - google_generate_content_endpoint_testing:
-          filters:
-            branches:
-              only: main
-      - ocr_testing:
-          filters:
-            branches:
-              only: main
-      - search_testing:
-          filters:
-            branches:
-              only: main
-      - batches_testing:
-          filters:
-            branches:
-              only: main
-      - litellm_utils_testing:
-          filters:
-            branches:
-              only: main
-      - pass_through_unit_testing:
-          filters:
-            branches:
-              only: main
-      - image_gen_testing:
-          filters:
-            branches:
-              only: main
-      - logging_testing:
-          filters:
-            branches:
-              only: main
-      - audio_testing:
-          filters:
-            branches:
-              only: main
+      - google_generate_content_endpoint_testing
+      - ocr_testing
+      - search_testing
+      - batches_testing
+      - litellm_utils_testing
+      - pass_through_unit_testing
+      - image_gen_testing
+      - logging_testing
+      - audio_testing
       - upload-coverage:
-          filters:
-            branches:
-              only: main
           requires:
             - realtime_translation_testing
             - google_generate_content_endpoint_testing
@@ -3505,27 +3442,12 @@ workflows:
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-      - installing_litellm_on_python:
-          filters:
-            branches:
-              only: main
-      - installing_litellm_on_python_3_13:
-          filters:
-            branches:
-              only: main
-      - installing_litellm_on_python_v2_migration_resolver:
-          filters:
-            branches:
-              only: main
+      - installing_litellm_on_python
+      - installing_litellm_on_python_3_13
+      - installing_litellm_on_python_v2_migration_resolver
       - helm_chart_testing:
-          filters:
-            branches:
-              only: main
           requires:
             - build_docker_database_image
       - test_bad_database_url:
-          filters:
-            branches:
-              only: main
           requires:
             - build_docker_database_image

.github/workflows/helm_unit_test.yml

@@ -22,12 +22,6 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Check Lens Compose configuration
-        run: |
-          python3 -m unittest discover -s deploy/lens -p 'test_*.py'
-          python3 deploy/lens/configure.py --version 1.2.3 --env-file "$RUNNER_TEMP/lens.env"
-          docker compose --env-file "$RUNNER_TEMP/lens.env" -f deploy/lens/stack.yaml config --quiet
-
       - name: Set up Helm 3.11.1
         uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
         with:

.github/workflows/image-scan.yml

@@ -17,10 +17,8 @@ on:
       - gateway/routes/allowlist.py
       - backend/Dockerfile
       - backend/main.py
-      - deploy/lens/**
       - litellm-rust/**
       - litellm/proxy/lens/**
-      - tests/e2e/migrations/lens_compose_smoke.sh
       - docker/component_entrypoint.sh
       - docker/entrypoint.sh
       - litellm/proxy/prisma_migration.py
@@ -47,57 +45,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  lens-worker-image:
-    name: lens-worker-image (${{ matrix.arch }})
-    runs-on: ${{ matrix.runner }}
-    if: >-
-      github.event_name != 'pull_request' ||
-      github.event.pull_request.head.repo.full_name == github.repository
-    timeout-minutes: 45
-    permissions:
-      contents: read
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-latest
-            grype_sha256: edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-            grype_sha256: 553e4c36d9d61349830ba6034d43b8700a7f10576d3e2f4981c0fd2b96086465
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - name: Build the release worker
-        env:
-          RELEASE_TAG: sha-${{ github.sha }}
-        run: docker build --build-arg LITELLM_RELEASE_TAG="${RELEASE_TAG}" -f deploy/lens/Dockerfile -t lens-worker-scan .
-      - name: Verify the standalone worker on a read-only filesystem
-        env:
-          RELEASE_TAG: sha-${{ github.sha }}
-        run: |
-          docker run --rm --network none --read-only --cap-drop ALL \
-            --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-            lens-worker-scan --version | grep -F "litellm-lens $RELEASE_TAG protocol="
-      - name: Download Grype v0.114.0
-        env:
-          ARCH: ${{ matrix.arch }}
-          GRYPE_SHA256: ${{ matrix.grype_sha256 }}
-        run: |
-          curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
-            "https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_${ARCH}.tar.gz"
-          echo "${GRYPE_SHA256}  $RUNNER_TEMP/grype.tar.gz" | sha256sum -c -
-          tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
-          chmod +x "$RUNNER_TEMP/grype"
-      - name: Scan the worker for fixable HIGH/CRITICAL CVEs
-        env:
-          GRYPE_MATCH_PYTHON_USING_CPES: "true"
-        run: |
-          "$RUNNER_TEMP/grype" lens-worker-scan \
-            --config .grype.yaml --only-fixed --fail-on high --output table
-
   image-scan:
     name: image-scan
     runs-on: ubuntu-latest
@@ -200,12 +147,6 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
 
-      - name: Verify the bundled Lens Compose installation and restart
-        if: matrix.dockerfile == 'Dockerfile'
-        env:
-          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
-        run: bash tests/e2e/migrations/lens_compose_smoke.sh
-
   migrations-image:
     name: migrations-image
     runs-on: ubuntu-latest

.github/workflows/lens-install-smoke.yml

@@ -1,94 +0,0 @@
-name: Lens installation smoke
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: lens-install-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-images:
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-          - component: backend
-            dockerfile: backend/Dockerfile
-          - component: ui
-            dockerfile: ui/Dockerfile
-          - component: migrations
-            dockerfile: migrations/Dockerfile
-          - component: monolith
-            dockerfile: Dockerfile
-          - component: worker
-            dockerfile: deploy/lens/Dockerfile
-    env:
-      COMPONENT: ${{ matrix.component }}
-      DOCKERFILE: ${{ matrix.dockerfile }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - name: Build the matching release image
-        run: |
-          docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci \
-            -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:v0.0.0-lens-ci" .
-      - name: Save the matching release image
-        run: |
-          docker save "lens-ci-$COMPONENT:v0.0.0-lens-ci" \
-            | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT.tar.gz"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: lens-install-${{ matrix.component }}-${{ github.sha }}
-          path: ${{ runner.temp }}/lens-install-${{ matrix.component }}.tar.gz
-          compression-level: 0
-          retention-days: 3
-          if-no-files-found: error
-          overwrite: true
-
-  helm-install:
-    needs: build-images
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-install-*-${{ github.sha }}
-          merge-multiple: true
-          path: ${{ runner.temp }}/lens-install-images
-      - name: Load the matching release images
-        run: |
-          for component in gateway backend ui migrations monolith worker; do
-            archive="$RUNNER_TEMP/lens-install-images/lens-install-$component.tar.gz"
-            gzip -dc "$archive" | docker load
-            rm "$archive"
-          done
-      - name: Install pinned Kubernetes test tools
-        run: |
-          curl --fail --location --output "$RUNNER_TEMP/kind" \
-            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
-          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kind"
-          curl --fail --location --output "$RUNNER_TEMP/kubectl" \
-            https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
-          echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea  $RUNNER_TEMP/kubectl" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kubectl"
-          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
-            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
-          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
-          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
-          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
-      - name: Install, ingest, upgrade, and restart both charts
-        run: bash tests/e2e/migrations/lens_helm_smoke.sh

.github/workflows/lens-worker.yml

@@ -1,111 +0,0 @@
-name: Lens Worker Image
-
-on:
-  pull_request:
-    branches: [main, litellm_oss_branch, "litellm_**"]
-    paths:
-      - deploy/lens/**
-      - litellm-rust/**
-      - litellm/proxy/lens/**
-      - tests/proxy_behavior/lens/**
-      - .github/workflows/lens-worker.yml
-  push:
-    branches: [main]
-    paths:
-      - deploy/lens/**
-      - litellm-rust/**
-      - litellm/proxy/lens/**
-      - tests/proxy_behavior/lens/**
-      - .github/workflows/lens-worker.yml
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lens-worker-image:
-    permissions:
-      contents: read
-      packages: write
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-latest
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - name: Build native Lens service
-        env:
-          RELEASE_TAG: sha-${{ github.sha }}
-        run: docker build --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-worker .
-      - name: Verify version and unprivileged runtime
-        env:
-          RELEASE_TAG: sha-${{ github.sha }}
-        run: bash deploy/lens/smoke.sh lens-worker "$RELEASE_TAG"
-      - name: Verify confined Python on the native architecture
-        env:
-          RELEASE_TAG: sha-${{ github.sha }}
-        run: |
-          docker build --target smoke --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-smoke .
-          docker run --rm --network none --read-only --cap-drop ALL \
-            --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges lens-smoke
-      - name: Reject custom builds without a matching release tag
-        run: |
-          if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
-            echo "::error::An unversioned worker build unexpectedly succeeded"
-            exit 1
-          fi
-          grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
-      - name: Publish development architecture
-        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
-        env:
-          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REGISTRY_USER: ${{ github.actor }}
-          IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}-${{ matrix.arch }}
-          ARCH: ${{ matrix.arch }}
-        run: |
-          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
-          docker tag lens-worker "$IMAGE"
-          docker push "$IMAGE"
-          mkdir -p digests
-          docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE" > "digests/$ARCH"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
-        with:
-          name: lens-digest-${{ matrix.arch }}
-          path: digests/
-          retention-days: 1
-
-  publish:
-    name: Publish Lens development index
-    needs: lens-worker-image
-    if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-    steps:
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-digest-*
-          merge-multiple: true
-          path: digests
-      - name: Publish both tested architectures
-        env:
-          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REGISTRY_USER: ${{ github.actor }}
-          IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}
-        run: |
-          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
-          docker buildx imagetools create --tag "$IMAGE" "$(cat digests/amd64)" "$(cat digests/arm64)"
-          printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test-unit.yml

@@ -139,6 +139,7 @@ jobs:
             test-path: |-
               tests/unit/decisions
               tests/unit/litellm_core_utils
+              tests/unit/systemone
             python-version: "3.12"
             workers: 4
             reruns: 0
@@ -343,6 +344,7 @@ jobs:
               tests/unit/llms/snowflake
               tests/unit/llms/soniox
               tests/unit/llms/stability
+              tests/unit/llms/strands_decider
               tests/unit/llms/tavily
               tests/unit/llms/tencent
               tests/unit/llms/tinyfish
@@ -668,6 +670,7 @@ jobs:
           - shard: proxy-extras
             test-path: >-
               tests/unit/litellm_proxy_extras
+              tests/unit/db_scripts
             python-version: "3.12"
             workers: 2
             reruns: 0

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -60,6 +60,7 @@ RUN apk add --no-cache \
     python-3.13-dev \
     rust \
     openssl \
+    libssl3 \
     openssl-dev \
     nodejs \
     npm \
@@ -119,8 +120,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
 FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
 COPY --from=uvbin /uv /usr/local/bin/uv
 RUN apk add --no-cache python-3.13
-ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
-    https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
+ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
+    https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
 RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
     uv venv /opt/liteadmin --python python3.13 && \
     uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@@ -140,7 +141,7 @@ USER root
 RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app

Makefile

@@ -4,7 +4,7 @@
 .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	test-rust-extension rust-sqlx-prepare lens-dev \
+	test-rust-extension rust-sqlx-prepare \
 	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-type-discipline \
 	lint-ruff-strict lint-gate lint-test-quality \
@@ -54,7 +54,6 @@ help:
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
 	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
-	@echo "  make lens-dev           - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large --seed-logs\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)"
 	@echo ""
 	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
 	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
@@ -290,9 +289,6 @@ test-rust-extension:
 rust-sqlx-prepare:
 	cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
 
-lens-dev:
-	./scripts/lens_dev.sh $(ARGS)
-
 test: install-test-deps
 	$(UV_RUN) pytest tests/
 

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -79,7 +79,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -94,7 +94,12 @@ ENV HOME=/home/nonroot \
     PYTHONUNBUFFERED=1 \
     PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
-COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
+COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
+COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
+COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
+COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
+COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

ci_cd/cost_map_guard.py

@@ -14,8 +14,9 @@ import argparse
 import json
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
@@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
 SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
 
@@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str:
         return str(error)
 
 
-def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
+def _evaluation_output_price_failure(
+    key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
+) -> str | None:
+    if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
+        return None
+    output_cost: Final = entry.get("output_cost_per_token")
+    if (
+        not isinstance(output_cost, (int, float))
+        or isinstance(output_cost, bool)
+        or output_cost <= 0
+        or key in evaluation_output_price_sources
+    ):
+        return None
+    return (
+        f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
+        "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
+    )
+
+
+def _file_failures(
+    head: Snapshot,
+    head_map: CostMap,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     schema_text: Final = _rendered_schema(head_map)
     if not schema_text.startswith("{"):
         return (schema_text,)
@@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
             f"{COST_MAP_PATH} does not validate against its schema: {error}"
             for error in validation_errors(head_map, json.loads(schema_text))[:20]
         ),
+        *(
+            failure
+            for key, entry in head_map.items()
+            if (
+                failure := _evaluation_output_price_failure(
+                    key, entry, evaluation_output_price_sources
+                )
+            )
+            is not None
+        ),
     )
 
 
@@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
-def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+def guard_failures(
+    base: Snapshot,
+    head: Snapshot,
+    changed_files: Sequence[str],
+    bot: bool,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     if not bot and not touches_cost_map(changed_files):
         return ()
     head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
     if isinstance(head_map, str):
         return (head_map,)
-    return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
+    return (
+        *_file_failures(
+            head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
+        ),
+        *(_bot_failures(base, head_map, changed_files) if bot else ()),
+    )
 
 
 def _git(*args: str) -> str | None:

ci_cd/generate_model_prices_schema.py

@@ -19,6 +19,7 @@ NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
 NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
 BOOLEAN: JsonSchema = {"type": "boolean"}
 STRING: JsonSchema = {"type": "string"}
+NON_BLANK_STRING: Final[JsonSchema] = {"type": "string", "pattern": r"\S"}
 TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
 WEEKDAY_PATTERN: Final = (
     r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
@@ -61,6 +62,17 @@ OFF_PEAK_WINDOW: Final[JsonSchema] = {
 }
 
 OBJECT_KEYS: dict[str, JsonSchema] = {
+    "search_free_requests": {
+        "type": "object",
+        "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.",
+        "required": ["count", "period", "shared_group"],
+        "additionalProperties": False,
+        "properties": {
+            "count": {"type": "integer", "minimum": 0},
+            "period": {"type": "string", "enum": ["day", "month"]},
+            "shared_group": {"type": "string", "pattern": "\\S"},
+        },
+    },
     "off_peak_pricing": {
         "type": "object",
         "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
@@ -103,6 +115,17 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
         "type": "object",
         "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
     },
+    "token_count_model": {
+        "type": "object",
+        "required": ["model"],
+        "additionalProperties": False,
+        "properties": {
+            "model": NON_BLANK_STRING,
+            "region": NON_BLANK_STRING,
+            "litellm_provider": {"type": "string", "enum": ["bedrock", "bedrock_mantle"]},
+        },
+        "description": "Model that counts tokens for this one, its counting provider, and the AWS region its count API runs in.",
+    },
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
@@ -189,6 +212,10 @@ INTEGER_KEYS: dict[str, JsonSchema] = {
         **NONNEG_INTEGER,
         "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
     },
+    "token_count_framing": {
+        **NONNEG_INTEGER,
+        "description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
+    },
     "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
     "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
 }
@@ -244,6 +271,11 @@ def cost_schema(key: str) -> JsonSchema:
 
 def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
     return {
+        "token_count_tokenizer": {
+            "type": "string",
+            "description": "Tokenizer litellm counts this model's requests with when the provider cannot count them.",
+            "enum": ["kimi_k3"],
+        },
         "litellm_provider": {
             "type": "string",
             "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",

cosign.pub

@@ -1,4 +1,4 @@
 -----BEGIN PUBLIC KEY-----
-MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
-POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
+JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
 -----END PUBLIC KEY-----

db_scripts/strip_vector_store_internal_metadata.py

@@ -0,0 +1,201 @@
+import argparse
+import os
+import sys
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, repeat, takewhile
+from typing import Final, Literal, TextIO
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
+
+from litellm.vector_stores.utils import is_litellm_written_metadata_key
+
+DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+LIST_PAGE_SIZE: Final = "100"
+
+
+class VectorStore(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    metadata: dict[str, JsonValue] | None = None
+
+
+class VectorStorePage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    data: tuple[VectorStore, ...]
+    has_more: bool = False
+
+
+class OpenAIErrorBody(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    message: str = ""
+
+
+class OpenAIErrorResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    error: OpenAIErrorBody
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    api_key: str
+    api_base: str
+    vector_store_ids: tuple[str, ...]
+    extra_keys: frozenset[str]
+    apply: bool
+
+
+@dataclass(frozen=True, slots=True)
+class StoreResult:
+    store_id: str
+    matched_keys: tuple[str, ...]
+    status: Literal["dry_run", "updated", "failed"]
+    error: str = ""
+
+
+def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
+    return is_litellm_written_metadata_key(key) or key in extra_keys
+
+
+def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
+    return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
+
+
+def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
+    params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
+    return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
+
+
+def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
+    if page is None or not page.has_more or not page.data:
+        return None
+    return fetch_page(client, after=page.data[-1].id)
+
+
+def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
+    pages: Final = accumulate(
+        repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
+    )
+    for page in takewhile(lambda page: page is not None, pages):
+        yield from page.data
+
+
+def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
+    for store_id in store_ids:
+        yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
+
+
+def describe_http_error(response: httpx.Response) -> str:
+    try:
+        message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
+    except ValidationError:
+        return f"HTTP {response.status_code}"
+    return f"HTTP {response.status_code}: {message}"
+
+
+def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
+    try:
+        response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
+    except httpx.TransportError as error:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
+    if response.is_success:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
+    return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
+
+
+def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
+    keys: Final = matching_keys(store, options.extra_keys)
+    if not keys:
+        return None
+    if not options.apply:
+        return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
+    return delete_keys(client, store.id, keys)
+
+
+def format_result(result: StoreResult) -> str:
+    key_list: Final = ", ".join(result.matched_keys)
+    count: Final = len(result.matched_keys)
+    match result.status:
+        case "dry_run":
+            return f"{result.store_id}: would delete {count} keys: {key_list}"
+        case "updated":
+            return f"{result.store_id}: deleted {count} keys: {key_list}"
+        case "failed":
+            return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
+
+
+def clean_and_report(
+    client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
+) -> Iterator[StoreResult | None]:
+    for store in stores:
+        result: Final = clean_store(client, store, options)
+        if result is not None:
+            print(format_result(result), file=out)
+        yield result
+
+
+def run(options: Options, client: httpx.Client, out: TextIO) -> int:
+    stores: Final = (
+        retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
+    )
+    scanned: Final = tuple(clean_and_report(client, stores, options, out))
+    results: Final = tuple(result for result in scanned if result is not None)
+    updated: Final = sum(1 for result in results if result.status == "updated")
+    failed: Final = sum(1 for result in results if result.status == "failed")
+    mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
+    print(
+        f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
+        file=out,
+    )
+    return 1 if failed else 0
+
+
+def parse_options(argv: Sequence[str] | None) -> Options:
+    parser: Final = argparse.ArgumentParser(
+        description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
+    )
+    parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE)
+    parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
+    parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
+    parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
+    args: Final = parser.parse_args(argv)
+    if not args.api_key:
+        parser.error("--api-key or OPENAI_API_KEY is required")
+    return Options(
+        api_key=args.api_key,
+        api_base=args.api_base.rstrip("/"),
+        vector_store_ids=tuple(args.vector_store_id),
+        extra_keys=frozenset(args.extra_key),
+        apply=args.apply,
+    )
+
+
+def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
+    options: Final = parse_options(argv)
+    with httpx.Client(
+        base_url=options.api_base,
+        headers={"Authorization": f"Bearer {options.api_key}"},
+        timeout=60.0,
+        transport=transport,
+    ) as client:
+        try:
+            return run(options, client, sys.stdout)
+        except httpx.HTTPStatusError as error:
+            sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
+            return 2
+        except httpx.TransportError as error:
+            sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
+            return 2
+        except ValidationError:
+            sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
+            return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

deploy/lens/Dockerfile

@@ -1,41 +0,0 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-
-FROM $LITELLM_BUILD_IMAGE AS builder
-RUN apk add --no-cache rust build-base cmake perl pkgconf openssl-dev libseccomp-dev python-3.13
-WORKDIR /src
-COPY .cargo/ .cargo/
-COPY litellm-rust/ litellm-rust/
-COPY litellm/proxy/lens/prompts/ litellm/proxy/lens/prompts/
-WORKDIR /src/litellm-rust
-ENV CARGO_PROFILE_RELEASE_DEBUG=0 CARGO_PROFILE_RELEASE_STRIP=symbols
-RUN cargo build --locked --release -p litellm-lens
-COPY deploy/lens/python_policy.c /tmp/python_policy.c
-RUN cc -std=c11 -D_GNU_SOURCE -O2 -Wall -Wextra -Werror /tmp/python_policy.c -lseccomp -o /tmp/python-policy && \
-    /tmp/python-policy /tmp/python.seccomp
-
-FROM builder AS test-builder
-RUN cargo test --locked --release -p litellm-lens --test sandbox --no-run --message-format=json > /tmp/test-artifacts.json && \
-    python3.13 -c 'import json, pathlib, shutil; rows = [json.loads(line) for line in pathlib.Path("/tmp/test-artifacts.json").read_text().splitlines()]; artifact, = [r["executable"] for r in rows if r.get("executable") and r["target"]["name"] == "sandbox"]; shutil.copyfile(artifact, "/tmp/lens-sandbox-tests")' && \
-    chmod 755 /tmp/lens-sandbox-tests
-
-FROM $LITELLM_RUNTIME_IMAGE AS service
-ARG LITELLM_RELEASE_TAG=""
-RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
-RUN apk add --no-cache python-3.13 setpriv libgcc libstdc++ openssl ca-certificates
-ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} PYTHONDONTWRITEBYTECODE=1
-WORKDIR /app
-COPY --from=builder /src/litellm-rust/target/release/litellm-lens /usr/local/bin/litellm-lens
-COPY --from=builder /tmp/python.seccomp /app/lens/python.seccomp
-COPY deploy/lens/python_runtime.py /tmp/python_runtime.py
-RUN python3.13 -S /tmp/python_runtime.py /app/lens/python-runtime.json && rm /tmp/python_runtime.py
-USER 65532:65532
-EXPOSE 4318
-ENTRYPOINT ["/usr/local/bin/litellm-lens"]
-
-FROM service AS smoke
-COPY --from=test-builder /tmp/lens-sandbox-tests /usr/local/bin/lens-sandbox-tests
-ENTRYPOINT ["/usr/local/bin/lens-sandbox-tests"]
-CMD ["--ignored", "--nocapture", "--test-threads=1"]
-
-FROM service AS runtime

deploy/lens/Dockerfile.dockerignore

@@ -1,15 +0,0 @@
-**
-!deploy/
-!deploy/lens/
-!deploy/lens/python_policy.c
-!deploy/lens/python_runtime.py
-!litellm/
-!litellm/proxy/
-!litellm/proxy/lens/
-!litellm/proxy/lens/prompts/
-!litellm/proxy/lens/prompts/**
-!.cargo/
-!.cargo/**
-!litellm-rust/
-!litellm-rust/**
-litellm-rust/target/

deploy/lens/README.md

@@ -1,282 +0,0 @@
-# Lens service
-
-Lens records agent activity and investigates it in a separate Rust service. LiteLLM serves model requests, the dashboard, and investigation settings. Lens owns trace ingestion and ClickHouse access; PostgreSQL stays with LiteLLM
-
-Agent exporters send traces directly to Lens. LiteLLM sends its optional request logs through a bounded background queue. If Lens or ClickHouse is unavailable, model requests continue; traces can be delayed or dropped according to the exporter's retry policy. The gateway never waits for ClickHouse during startup or inference
-
-## New local installation
-
-Install Docker with Compose, Python 3.10 or later, and Git. Clone LiteLLM, select a published release that includes Lens, and start the existing Compose stack:
-
-```bash
-git clone https://github.com/BerriAI/litellm.git
-cd litellm
-python3 deploy/lens/configure.py --version <release-version>
-docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml up -d --wait
-```
-
-The configuration command generates your keys and database passwords once, saves them in `deploy/lens/.env` with owner-only permissions, and preserves them on subsequent runs. Back up this file alongside your database volumes. Both images use the selected release; there is no local image build
-
-Open `http://localhost:4000/ui/` and sign in as `admin` using `LITELLM_MASTER_KEY` from the saved file. Open **Lens**, select your framework, generate a tracing key, and copy the displayed configuration. The trace endpoint is already filled in. Keep your agent's existing model credentials; the tracing key only authorizes trace uploads
-
-PostgreSQL and ClickHouse use persistent Docker volumes and have no host ports. The dashboard and trace listener bind to localhost. Use your normal TLS and ingress for a hosted deployment. Stop the stack with `docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml down`; omit `-v` to retain data
-
-Under **Lens > Investigations > Connect worker**, choose an analysis model and monthly budget. The deployed service connects automatically after you save these settings. There is no worker command or second token to copy
-
-## Existing LiteLLM installation
-
-Keep your gateway, PostgreSQL database, deployment tool, and existing encryption keys. Deploy the matching Lens image, give it access to ClickHouse, and configure the service connection on LiteLLM
-
-| Variable | LiteLLM | Lens service |
-| --- | --- | --- |
-| `LITELLM_LENS_SERVICE_TOKEN` | Same private random secret, at least 32 characters | Same secret |
-| `LITELLM_LENS_URL` | Internal Lens URL, such as `http://lens-worker:4318` | Not needed |
-| `LITELLM_LENS_PUBLIC_URL` | Ingestion base URL reachable by your agents | Not needed |
-| `LITELLM_URL` | Not needed | LiteLLM URL reachable from Lens |
-| `CLICKHOUSE_URL` | Remove it from Lens tracing configuration | ClickHouse HTTP URL with credentials |
-| `CLICKHOUSE_DATABASE` | Not needed for Lens | Existing database name, defaults to `litellm` |
-| `AGENT_TRACING_RETENTION_DAYS` | Not needed for Lens | Retention for traces and Lens request logs, defaults to `14` |
-
-Remove the old `general_settings.tracing.store` configuration used for Lens from LiteLLM. Keep unrelated logging integrations and their configuration. Only Lens should reach its ClickHouse database. The shared service secret is an infrastructure credential: keep it out of browser code, agent exporters, screenshots, and public ingress headers
-
-Expose the Lens HTTP listener on port 4318 through TLS. Route `/lens-ingest` on your existing hostname directly to Lens at the load balancer, then set `LITELLM_LENS_PUBLIC_URL=https://<your-host>/lens-ingest`. The gateway must not proxy these uploads. Alternatively use a separate hostname and forward `/v1/` to Lens. Keep `/internal/` private; it requires the service secret
-
-### Standalone Docker or a container host
-
-Build from the same source commit and `LITELLM_RELEASE_TAG` as your running gateway:
-
-```bash
-export LITELLM_RELEASE_TAG='<gateway-release-identity>'
-export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<image-tag>'
-docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
-  -f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
-```
-
-Publish that image to a registry your host can pull from. Prefer a digest reference for hosted deployments. Public development images use `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`; check that the exact image exists before selecting it. An arbitrary commit may not have a published image
-
-The image supports native amd64 and arm64. For worker-only Compose, use `deploy/lens/compose.yaml` with a private environment file containing `LENS_WORKER_IMAGE`, `LITELLM_URL`, `LITELLM_LENS_SERVICE_TOKEN`, and `CLICKHOUSE_URL`:
-
-```bash
-docker compose --env-file /path/to/private/lens.env \
-  -f deploy/lens/compose.yaml up -d
-```
-
-The Compose listener binds to localhost. Your reverse proxy must reach it. On Render, run Lens as a web service with the same environment and listener port 4318, not an outbound-only background worker. Use `/health/live` for process health and `/health/ready` to check storage and tracing credentials
-
-Lens does not need provider credentials, PostgreSQL credentials, a GPU, or the LiteLLM Python package. The image includes a small CPython runtime only for the investigator's confined calculation tool. Keep the shipped security settings, temporary filesystem, and resource limits
-
-### Kubernetes with Helm
-
-Both `helm/litellm` and `helm/litellm-helm` support Lens. Keep your existing chart, release name, namespace, and values. Add:
-
-```yaml
-lensWorker:
-  enabled: true
-```
-
-Then run your usual Helm deployment command using the matching published chart. The chart supplies the matching Lens image, generates the shared service secret, starts a single ClickHouse instance with a persistent volume, and connects the services. Your cluster needs a default storage class, or set `lensWorker.clickhouse.storageClassName`. Bundled storage defaults to 20 GiB; set `lensWorker.clickhouse.storage` before installation to choose another size
-
-When your chart manages an ingress with one hostname, the chart fills in the public tracing address and routes `/lens-ingest` directly to Lens. TLS is detected from `ingress.tls` or an ALB certificate annotation. With custom ingress, multiple hostnames, or TLS terminated elsewhere, set the address explicitly:
-
-```yaml
-lensWorker:
-  enabled: true
-  publicUrl: https://<your-litellm-host>/lens-ingest
-```
-
-For a dedicated trace hostname, configure `lensWorker.ingress.enabled`, `host`, `className`, and `tls`. Its hostname supplies the public address unless you override `publicUrl`. Internal Lens routes stay private
-
-To use an existing ClickHouse database and secrets managed by your platform, keep these overrides:
-
-```yaml
-lensWorker:
-  enabled: true
-  serviceTokenSecret:
-    name: litellm-lens-service
-    key: service-token
-  clickhouseSecret:
-    name: litellm-lens-clickhouse
-    key: url
-  clickhouseDatabase: litellm
-  retentionDays: 14
-```
-
-Supplying `clickhouseSecret.name` uses that database and disables bundled storage. Keep your database name and retention policy. For GitOps tools that render Helm without cluster access, supply both existing secrets so rendering cannot regenerate credentials
-
-Normal Helm upgrades reuse the generated credentials. Secrets are retained on uninstall, and the ClickHouse volume is retained by Kubernetes. Back them up together. Treat changing the database, storage class, or secret reference as an infrastructure change, not a routine version update
-
-After deployment, open **Lens**. If it was already open, click **Check setup**. The setup section moves to your framework and tracing key when Lens is reachable and storage is ready. Investigation setup asks for the analysis model and budget; the installed service connects automatically
-
-## Upgrade
-
-Upgrade LiteLLM and Lens from the same source commit and release identity. For a coordinated published release, use its matching worker version; `deploy/lens/stack.yaml` starts LiteLLM, Lens, PostgreSQL, and ClickHouse for new installations. Standalone images remain available. Publishing an image does not update running containers
-
-Keep the same databases, encryption keys, shared service secret, and public ingestion URL. Pause scheduled investigations and finish or cancel active runs, update both images through your usual deployment process, then check ingestion and run an investigation before resuming schedules. Do not run `docker compose down -v`
-
-## Development
-
-`make lens-dev` starts LiteLLM, Lens, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work
-
-## Configure a lens
-
-Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute
-
-Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage
-
-Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Each scan recalculates the window and reuses completed reviews when the selected trace content, expected behavior, enabled checks and analysis model are unchanged. Budget, name and schedule edits preserve reuse. Duplicate a lens when you want a separate investigation without changing an existing monitor
-
-Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 2 to 15 seconds, backing off while idle; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
-
-## Read the results
-
-Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
-
-Use the **Investigation run** selector or **History** to reopen previous results. Each run keeps its new or updated findings, settings, selected traces, coverage and cost. An unchanged rerun adds no findings; choose **All accumulated findings** to see saved findings across runs. The **Agent traces** tab lists the selected sample, including traces without an observed issue and traces with insufficient evidence. The tab is called **LLM requests** or **Traces and requests** for those activity types. Findings distinguish distinct affected traces from contributing investigation runs. Counterexamples remain visible as evidence without increasing the affected count. Merged finding links continue to resolve to the retained finding
-
-Enter an explanation under **What should Lens remember?** and choose **This is expected** to dismiss expected behavior, **Mark resolved** after fixing an issue, or **Reopen** to reopen a resolved issue. These actions save the feedback together with the status; typing feedback alone neither saves it nor resolves the finding. Feedback informs later analysis and reconciliation without invalidating completed reviews. It stays with the finding when evidence recurs, does not alter historical evidence, and does not exempt different problems
-
-## What a scan does
-
-The proxy selects executions received or updated within the configured lookback window, with a two-minute settling period. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID
-
-A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
-
-The worker reads complete selected trace content to compute fingerprints before making paid model calls. Completed review checkpoints are reused only within the same lens when both the complete content and investigation criteria match. Criteria are the expected behavior, enabled check IDs and instructions, and analysis model. Lens fingerprints these values rather than using the time of an unrelated settings edit. Scope and sampling changes preserve matching reviews for traces selected again; duplicating a lens starts an independent set of reviews. Initial reviews are confined to their assigned trace and retain observations, cited excerpts and metadata. Reviewers use catalog, read, search and optional Python tools; Python receives selected evidence as streamed input. Pending observation batches are grouped in parallel and investigated against the original evidence, then reconciled with saved findings. A recurring cause extends its existing finding, preserving feedback, earlier evidence and contributing run history
-
-If every selected review is already incorporated into findings, the run completes with a reuse count, zero model calls and zero analysis cost, including when the monthly budget is exhausted. The run remains in history. A partially failed run preserves completed review checkpoints; pending grouping or investigation can still need paid model calls even when all selected trace reviews are reused. A trace without a complete matching checkpoint needs a review. Live progress distinguishes reviews eligible for reuse from reviews actually recorded; failed or cancelled runs report only the reuse they completed
-
-There is no fixed total run, span, candidate or investigation-turn cutoff. Agents can replace their active conversation with working notes. If a request exceeds the configured model's context window, the worker compacts the conversation automatically and resumes with references to its archived tool history. Original evidence remains accessible through the gateway while it is available and retained. Tool results and working notes remain accessible during the investigation; character ranges make even a single oversized result readable in pieces. A review reports an error if the task or its replacement notes cannot fit. Context windows, the configured budget, worker resources and recorded evidence still bound practical work. The investigator has no browsing, code-editing or production-action tools
-
-The live review drawer shows loading, trace review, parallel grouping, reconciliation and candidate investigation. It reports current model and tool operations, including context compaction, and retains tool-call counts on completed trace reviews. These counts describe attempted calls, not successful executions. This progress channel contains operation metadata, not Python code or tool output. Preliminary observations remain separate from final findings and their validated evidence
-
-Each model response must match its JSON schema. A malformed response gets one repair attempt through the same budget controls. A session review that remains invalid or cannot fit marks that execution unassessable while other reviews continue. Broken evidence pagination or missing content pages return tool errors so the agent can inspect narrower spans or other evidence. Unreadable citations receive repair feedback. Verified excerpts remain available without fetching their source again. The affected source counts as partial, including failures discovered during later investigations, while the reviewer owns its assessment. Findings are published only after comparison with each other and saved findings finishes. If analysis stops before that comparison completes, completed trace reviews and their evidence remain saved for reuse, and the run retains its assessments and error. A later run can retry grouping and investigation. Runs with useful completed assessments or reconciled findings show partial results; total failures are marked failed. Transport errors, cancellation and budget exhaustion stop further analysis. Both the worker and proxy validate quoted evidence against original content. Per-run issue assessments follow supporting citations, including evidence found by another run's reviewer; counterexamples do not mark a run affected. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
-
-Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
-
-## Operations and limits
-
-PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
-
-Lens checks which selected traces have reusable reviews before requesting model budget. Reuse needs no model call or reservation. New trace reviews and unfinished grouping or investigation can incur cost
-
-Before each model call, Lens reserves a conservative allowance based on the input and permitted output. The summary separates settled monthly spend, unexpired reservations and available budget. With a $100 limit, $45 spent and $10 reserved, $45 is available for additional calls. Successful calls settle to recorded cost and release unused capacity. Failed or timed-out requests release their hold; abandoned holds expire after the proxy request timeout plus a grace period
-
-Calls wait when concurrent reservations temporarily hold the remaining capacity. Waiting and model execution share the proxy's request timeout. If one request's allowance exceeds the unspent monthly budget, the error reports what the request needs and what remains. Reduce the deployment's output allowance or increase the limit. Paid analysis stops when the monthly limit is spent; completed reviews can still be reused. The monthly budget renews on the UTC calendar month
-
-The assigned virtual key has independent budgets, model permissions and rate limits. Several investigations can share that key, so its limit can stop analysis even when one lens has budget left. Every worker needs a billing key assigned through worker setup or **Settings**. Analysis spend appears under that key in **Virtual Keys** and normal request logs, with Lens, run and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Terminal budget, authentication or transport failures stop the scan after applicable retries and preserve completed checkpoints
-
-Lens requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Changed content requires a matching review before reuse. Findings should be reviewed by a person before acting on them
-
-
-## API access
-
-The UI and API use the same scan lifecycle. Authenticate with a proxy administrator credential for writes, or a proxy-admin viewer credential for reads. Worker credentials are only for worker operations
-
-```bash
-curl "$LITELLM_URL/lens" -H "Authorization: Bearer $LITELLM_API_KEY" \
-  -H 'Content-Type: application/json' -d '{
-    "name": "Research quality", "model": "your-model-alias",
-    "context": "Answer the requested question using cited, retrieved evidence.",
-    "source": "traces", "lookback_hours": 24,
-    "sample_percent": 100, "sample_size": null, "concurrency": 8,
-    "enabled": true, "interval_minutes": 1440, "monthly_budget": 50
-  }'
-
-curl "$LITELLM_URL/lens/$LENS_ID/runs" -X POST \
-  -H "Authorization: Bearer $LITELLM_API_KEY" -H 'Content-Type: application/json' -d '{}'
-
-curl "$LITELLM_URL/lens/$LENS_ID/runs?offset=0" -H "Authorization: Bearer $LITELLM_API_KEY"
-curl "$LITELLM_URL/lens/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITELLM_API_KEY"
-```
-
-Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/lens/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /lens/{id}/findings/{finding_id}` with `status` and `reason`
-
-## Local development
-
-`make lens-dev ARGS=--seed` starts the full dev stack. The live dashboard is at `http://localhost:3000/ui/lens/`, with login at `http://localhost:3000/ui/login/`. Next.js forwards API requests to the proxy on port 4000, so login and navigation stay in the live UI and edits hot-reload
-
-The default is Next.js dev with no production build (`LENS_DEV_BUILD_UI=0`). Set `LENS_DEV_BUILD_UI=1` when you also want a fresh static dashboard at `http://localhost:4000/ui/`. Build output goes to `.lens-dev/logs/ui-build.log`; a failed build stops startup. Both modes keep the live dashboard on port 3000. Startup checks the live login route before seeding and fails with the UI log path if Next.js exits. `LENS_DEV_STARTUP_TIMEOUT_SECONDS` controls startup readiness retries (default 300; `LENS_DEV_READINESS_REQUEST_TIMEOUT_SECONDS` caps each HTTP probe, default 5)
-
-For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies spread over the last 24 hours, about 860,000 spans with linked request logs, plus three long sessions of roughly 1,150, 9,200 and 92,000 spans in a single trace for drawer paging and the oversized read path. Their trace IDs are printed at the end. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. Every profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks, and verifies linked spend totals through the proxy. Large seeds then copy that first copy inside ClickHouse and PostgreSQL with `INSERT ... SELECT`, rewriting trace, span and call IDs so each copy keeps its own spend, and verify the last copy through the proxy
-
-Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
-
-The Rust receiver bounds each upload and its decompressed body to 16 MiB and permits two ingestion requests at once per replica. Exporters should split large batches and retry backpressure. `LENS_DEV_SEED_COPIES` and `LENS_DEV_SEED_TIMEOUT_SECONDS` control the seeder; the receiver's limits are compiled into the service
-
-## Quality evaluation
-
-Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
-
-```bash
-cargo build --manifest-path litellm-rust/Cargo.toml -p litellm-lens --example worker_once --locked
-python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
-  --model your-model-alias --split all --background 1000 --concurrency 16 \
-  --output /tmp/lens-quality.json
-```
-
-Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
-
-The default workspace retrieves trace content on demand. Python calls have temporary scratch space that is removed after execution. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
-
-To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
-
-## Release compatibility
-
-Gateway and worker builds carry the same `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched
-
-PostgreSQL stores complete review checkpoints in `LiteLLM_LensReview`, alongside Lens records and run history. Schema migrations preserve saved investigations, findings, history, worker credentials and billing assignments without rewriting stored Lens records
-
-Drain active scans, stop workers, back up the database, and deploy all gateway replicas as a coordinated replacement or traffic cutover. Keep traffic paused until every gateway replica uses the selected build and its migrations have completed. Mixed gateway versions sharing Lens data are not supported because every replica must understand the stored records. Pausing workers alone does not prevent dashboard or API writes. Recreate workers with the matching image and their existing tokens, then resume traffic and schedules. Scanning waits until a compatible worker connects
-
-Use the shipped migration history for databases containing Lens data. The schema guard stops `--use_prisma_db_push` before changes if it detects Lens tables that require renaming; run the shipped rename migration against the configured schema before retrying. Fresh databases and databases with the current table names can use database push
-
-A rollback to a gateway that cannot read saved Lens records requires restoring a compatible database backup. Database push from such a build can also remove the review table. Test recovery on a separate database and account for all gateway data written after the backup
-
-The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Set an explicit `LENS_WORKER_IMAGE` for worker-only Compose. Verify that the image exists and matches the gateway before deploying it
-
-For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version
-
-The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes to `ghcr.io/berriai/litellm-lens-worker-dev` on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker
-
-
-## Worker dependencies
-
-The service builds from the workspace Cargo.lock with a pinned Rust toolchain and a digest-pinned Wolfi runtime. It has no Python package dependencies. CPython and libseccomp support the confined calculation tool. CI builds, runs, and scans native amd64 and arm64 images
-
-## Python analysis boundary
-
-The `python` tool runs ordinary CPython with the standard library in a fresh child process inside the existing worker container. It receives the selected evidence as `data` over stdin and has its own temporary working directory. It creates no additional container or service. Read and search tools remain available independently of Python
-
-The native worker image builds a syscall policy with libseccomp and includes the full `setpriv` launcher. Each child starts with no inherited worker secrets or open worker files, isolated Python startup, Landlock filesystem restrictions and a default-deny seccomp filter. It can read the Python runtime and its own scratch files. Worker source, installed worker packages, other jobs' files and `/proc` contents are unavailable. Network sockets, child processes, cross-process memory operations, signals to other processes and filesystem metadata mutation are denied, including calls made through `ctypes`. Some metadata inspection, such as `stat`, `access` and `readlink` of known paths, remains possible
-
-Python execution requires a native Linux worker with Landlock ABI 3 or later and seccomp filtering. Build the image for the host architecture. Missing policy files, an incompatible kernel, or an unsupported host such as a macOS source worker returns a clear tool error. There is no unrestricted execution fallback. Keep the container's non-root user, dropped capabilities, no-new-privileges setting, read-only root and writable temporary mount
-
-The worker permits two Python children at once across all investigations. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
-
-| Per-call resource | Default |
-| --- | --- |
-| Elapsed execution time | 60 seconds |
-| CPU time | 30 seconds |
-| Process address space | 512 MiB |
-| Captured stdout or stderr | 4 MiB per stream |
-| Individual scratch file size | 16 MiB |
-| Monitored scratch storage | 64 MiB |
-| Monitored scratch entries | 2,048 |
-| Scratch directory depth | 128 |
-| Open file descriptors | 64 |
-
-Evidence is streamed from gateway pages into the confined child without building another complete selection in worker memory. The child decodes the selected data under its memory limit before running the code. The execution wall clock starts after input delivery; gateway fetches keep their HTTP timeouts and remain cancellable. CPU, address-space and file-size limits apply during input decoding as well as computation. Scratch usage is monitored every 50 milliseconds, so a call can temporarily overshoot its scratch allowance. The worker's shared temporary mount supplies the hard aggregate storage ceiling, 1 GiB by default. Accounting includes unlinked open files and files retained only by memory mappings. A mapped scratch inode without an open descriptor or directory entry is conservatively charged at the individual file-size limit, which may overcount small files. Cancellation and limit failures kill and reap the child before removing its scratch directory
-
-Results include `stdout`, `stderr`, `exit_code`, `error` and `output_complete`. Nonzero interpreter exits, confinement failures and resource failures set `error` and `output_complete=false`. Available traceback output is retained. An output-size failure delivers no partial stdout/stderr; the agent can narrow its computation and retry. A successful result retains all captured output without truncation
-
-This is a process boundary sharing the worker's Linux kernel. The checked-in smoke test verifies useful Python operations, filesystem and process restrictions, raw syscall attempts, resource failures, mapping accounting, cleanup and cancellation in the actual image. Run it on the deployment's native architecture and kernel:
-
-```bash
-docker build --target smoke --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-  -f deploy/lens/Dockerfile -t lens-worker:smoke .
-docker run --rm --read-only --cap-drop ALL \
-  --security-opt no-new-privileges --network none \
-  --tmpfs /tmp:rw,noexec,nosuid,size=1g lens-worker:smoke
-```
-
-The smoke target runs the Rust sandbox integration tests. The production image contains neither Cargo nor the test executable

deploy/lens/compose.build.yaml

@@ -1,8 +0,0 @@
-services:
-  lens-worker:
-    build:
-      context: ../..
-      dockerfile: deploy/lens/Dockerfile
-      args:
-        LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?Set the release tag used by the gateway}
-    image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -1,21 +0,0 @@
-services:
-  lens-worker:
-    image: ${LENS_WORKER_IMAGE:-${LITELLM_VERSION:+ghcr.io/berriai/litellm-lens-worker:v}${LITELLM_VERSION:-}}
-    environment:
-      LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
-      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
-      LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the same secret on LiteLLM and Lens}
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:?Set the ClickHouse URL reachable from Lens}
-      CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
-      AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
-    ports:
-      - "127.0.0.1:${LENS_PORT:-4318}:4318"
-    mem_limit: 2g
-    cpus: 2
-    pids_limit: 64
-    restart: unless-stopped
-    read_only: true
-    tmpfs:
-      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
-    cap_drop: [ALL]
-    security_opt: [no-new-privileges:true]

deploy/lens/config.yaml

@@ -1,7 +0,0 @@
-model_list: []
-
-general_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
-  tracing:
-    store:
-      type: lens

deploy/lens/configure.py

@@ -1,73 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import os
-import re
-import secrets
-import shlex
-import sys
-from pathlib import Path
-from typing import Final
-
-SECRET_NAMES: Final = (
-    "LITELLM_MASTER_KEY",
-    "LITELLM_SALT_KEY",
-    "LITELLM_LENS_SERVICE_TOKEN",
-    "POSTGRES_PASSWORD",
-    "CLICKHOUSE_PASSWORD",
-)
-
-
-def environment_content(path: Path) -> str:
-    if not path.exists():
-        return "".join(
-            f"{name}={'sk-' if name.endswith('KEY') else ''}{secrets.token_hex(32)}\n" for name in SECRET_NAMES
-        )
-    saved: Final = path.read_text().splitlines()
-    values: Final = dict(line.split("=", 1) for line in saved if "=" in line)
-    if any(not values.get(name) for name in SECRET_NAMES):
-        raise ValueError(f"{path} is incomplete. Restore your saved credentials before continuing")
-    return "\n".join(line for line in saved if not line.startswith("LITELLM_VERSION=")) + "\n"
-
-
-def configure(path: Path, version: str) -> None:
-    release: Final = version.removeprefix("v")
-    if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-.][a-zA-Z0-9.-]+)?", release):
-        raise ValueError("Use a published release version, such as 1.82.0 or 1.82.0-nightly")
-    if path.is_symlink():
-        raise ValueError(f"Refusing to replace a symlink: {path}")
-    existing: Final = path.exists()
-    content: Final = environment_content(path)
-    temporary: Final = path.with_name(f".{path.name}.{secrets.token_hex(8)}")
-    descriptor: Final = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
-    try:
-        with os.fdopen(descriptor, "w") as output:
-            output.write(content + f"LITELLM_VERSION={release}\n")
-        if existing:
-            os.replace(temporary, path)
-        else:
-            os.link(temporary, path)
-    finally:
-        temporary.unlink(missing_ok=True)
-
-
-def main() -> None:
-    parser: Final = argparse.ArgumentParser(description="Create or update the configuration for the Lens Compose stack")
-    parser.add_argument("--version", required=True, help="Published LiteLLM release; Lens uses the matching version")
-    parser.add_argument("--env-file", type=Path, default=Path(__file__).with_name(".env"))
-    arguments: Final = parser.parse_args()
-    try:
-        configure(arguments.env_file, arguments.version)
-    except (OSError, ValueError) as error:
-        parser.exit(1, f"Could not configure Lens: {error}\n")
-    sys.stdout.write(
-        f"Saved {arguments.env_file}. Existing keys and database passwords are preserved\n"
-        f"Start with: docker compose --env-file {shlex.quote(str(arguments.env_file))} "
-        "-f deploy/lens/stack.yaml up -d --wait\n"
-        "Open http://localhost:4000/ui/ and sign in as admin with LITELLM_MASTER_KEY from the saved file\n"
-        "Back up this file with your database volumes. Do not commit it\n"
-    )
-
-
-if __name__ == "__main__":
-    main()

deploy/lens/python_policy.c

@@ -1,63 +0,0 @@
-#include <errno.h>
-#include <fcntl.h>
-#include <seccomp.h>
-#include <stdio.h>
-#include <sys/ioctl.h>
-#include <unistd.h>
-
-static int allow(scmp_filter_ctx policy, const char *name)
-{
-    int number = seccomp_syscall_resolve_name(name);
-    return number < 0 ? 0 : seccomp_rule_add(policy, SCMP_ACT_ALLOW, number, 0);
-}
-
-int main(int argc, char **argv)
-{
-    const char *calls[] = {
-        "read", "write", "readv", "writev", "pread64", "pwrite64", "close", "close_range",
-        "open", "openat", "openat2", "fstat", "stat", "lstat", "newfstatat", "statx",
-        "lseek", "getdents", "getdents64", "access", "faccessat", "faccessat2",
-        "readlink", "readlinkat", "getcwd", "chdir", "fchdir", "statfs", "fstatfs",
-        "mkdir", "mkdirat", "rmdir", "unlink", "unlinkat", "rename", "renameat", "renameat2",
-        "link", "linkat", "symlink", "symlinkat", "truncate", "ftruncate", "fsync", "fdatasync",
-        "mmap", "mmap2", "mprotect", "munmap", "mremap", "madvise", "brk",
-        "rt_sigaction", "rt_sigprocmask", "rt_sigreturn", "rt_sigsuspend", "rt_sigtimedwait", "sigaltstack",
-        "getpid", "getppid", "gettid", "getuid", "geteuid", "getgid", "getegid", "getgroups",
-        "clock_gettime", "clock_getres", "clock_nanosleep", "gettimeofday", "time", "nanosleep",
-        "futex", "futex_time64", "set_tid_address", "set_robust_list", "rseq", "arch_prctl",
-        "sched_getaffinity", "sched_yield", "getrandom", "getrlimit", "setrlimit", "getrusage", "umask",
-        "dup", "dup2", "dup3", "pipe", "pipe2", "poll", "ppoll", "select", "pselect6",
-        "epoll_create", "epoll_create1", "epoll_ctl", "epoll_wait", "epoll_pwait", "epoll_pwait2",
-        "capget", "capset", "prctl", "landlock_create_ruleset", "landlock_add_rule", "landlock_restrict_self",
-        "execve", "exit", "exit_group", "uname", "sysinfo", "restart_syscall"
-    };
-    const int commands[] = {F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_SETFD, F_GETFL, F_GETLK, F_SETLK, F_SETLKW};
-    if (argc != 2) {
-        fputs("Usage: python-policy OUTPUT\n", stderr);
-        return 1;
-    }
-    scmp_filter_ctx policy = seccomp_init(SCMP_ACT_ERRNO(EPERM));
-    if (!policy)
-        return 1;
-    int result = 0;
-    for (size_t i = 0; i < sizeof(calls) / sizeof(calls[0]); i++)
-        result |= allow(policy, calls[i]);
-    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(prlimit64), 1, SCMP_A0(SCMP_CMP_EQ, 0));
-    for (size_t i = 0; i < sizeof(commands) / sizeof(commands[0]); i++)
-        result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 1, SCMP_A1(SCMP_CMP_EQ, commands[i]));
-    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 2,
-                              SCMP_A1(SCMP_CMP_EQ, F_SETFL), SCMP_A2(SCMP_CMP_MASKED_EQ, O_ASYNC, 0));
-    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIOCLEX));
-    result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIONCLEX));
-    int output = open(argv[1], O_WRONLY | O_CREAT | O_TRUNC, 0444);
-    if (output < 0)
-        result = -1;
-    if (!result)
-        result = seccomp_export_bpf(policy, output);
-    if (output >= 0)
-        close(output);
-    seccomp_release(policy);
-    if (result)
-        fputs("Could not build the Python syscall policy\n", stderr);
-    return result ? 1 : 0;
-}

deploy/lens/python_runtime.py

@@ -1,41 +0,0 @@
-import json
-import subprocess
-import sys
-import sysconfig
-from itertools import chain
-from pathlib import Path
-from typing import Final
-
-
-def dependencies(path: Path, loader: Path) -> tuple[Path, ...]:
-    result: Final = subprocess.run((str(loader), "--list", str(path)), capture_output=True, text=True, check=True)
-    if "not found" in result.stdout:
-        raise RuntimeError(f"Missing Python runtime library: {path}")
-    words: Final = tuple(result.stdout.split())
-    return tuple(Path(word).resolve() for word in words if word.startswith("/"))
-
-
-def main() -> None:
-    stdlib: Final = Path(sysconfig.get_path("stdlib")).resolve()
-    executable: Final = Path(sys.executable).resolve()
-    loaders: Final = tuple(Path("/usr/lib").glob("ld-linux-*.so.*"))
-    if len(loaders) != 1:
-        raise RuntimeError("Expected one native glibc dynamic loader in the Lens worker image")
-    entries: Final = tuple(
-        path for path in stdlib.iterdir() if path.name not in ("site-packages", "dist-packages", "__pycache__")
-    )
-    extensions: Final = tuple((stdlib / "lib-dynload").glob("*.so"))
-    libraries: Final = frozenset(
-        chain.from_iterable(dependencies(binary, loaders[0]) for binary in (executable, *extensions))
-    )
-    manifest: Final = {
-        "executable": str(executable),
-        "directories": (str(stdlib),),
-        "read": tuple(sorted(str(path) for path in {*entries, *libraries})),
-        "execute": tuple(sorted(str(path) for path in (executable, *loaders))),
-    }
-    Path(sys.argv[1]).write_text(json.dumps(manifest), encoding="utf-8")
-
-
-if __name__ == "__main__":
-    main()

deploy/lens/smoke.sh

@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-image="${1:?pass the built image reference}"
-release="${2:?pass the expected release tag}"
-version="$(docker run --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges "$image" --version)"
-test "$version" = "litellm-lens $release protocol=7"
-container="$(docker run -d --network none --read-only --cap-drop ALL \
-  --security-opt no-new-privileges --pids-limit 64 --memory 2g --cpus 2 \
-  --tmpfs /tmp:rw,noexec,nosuid,size=256m \
-  -e LITELLM_URL=http://127.0.0.1:1 \
-  -e CLICKHOUSE_URL=http://127.0.0.1:1 \
-  -e LITELLM_LENS_SERVICE_TOKEN=isolated-runtime-smoke-secret-32-characters \
-  "$image")"
-trap 'docker rm -f "$container" >/dev/null' EXIT
-test "$(docker exec "$container" id -u)" = 65532
-docker exec -i "$container" python3.13 -I -S - <<'PY'
-import time
-import urllib.error
-import urllib.request
-
-for attempt in range(50):
-    try:
-        with urllib.request.urlopen("http://127.0.0.1:4318/health/live", timeout=1) as response:
-            assert response.status == 200
-        break
-    except urllib.error.URLError:
-        if attempt == 49:
-            raise
-        time.sleep(0.1)
-
-for path, expected in (("health/ready", 503), ("internal/status", 401)):
-    try:
-        urllib.request.urlopen(f"http://127.0.0.1:4318/{path}", timeout=1)
-    except urllib.error.HTTPError as error:
-        assert error.code == expected, (path, error.code)
-    else:
-        raise AssertionError(f"{path} should return {expected}")
-print("Unprivileged Lens service remains live with unavailable dependencies")
-PY

deploy/lens/stack.yaml

@@ -1,100 +0,0 @@
-name: litellm-lens
-
-services:
-  litellm:
-    image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix}
-    entrypoint:
-      - python3
-      - -c
-      - |
-        import os, sys
-        from urllib.parse import quote
-        postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
-        os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
-        os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
-    command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
-    environment:
-      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key}
-      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
-      STORE_MODEL_IN_DB: "True"
-      LITELLM_LENS_URL: http://lens-worker:4318
-      LITELLM_LENS_PUBLIC_URL: ${LITELLM_LENS_PUBLIC_URL:-http://localhost:4318}
-      LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the shared Lens service secret}
-      LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
-    volumes:
-      - ./config.yaml:/app/lens-config.yaml:ro
-    ports:
-      - "127.0.0.1:${LITELLM_PORT:-4000}:4000"
-    networks: [proxy, database]
-    depends_on:
-      db:
-        condition: service_healthy
-    restart: unless-stopped
-
-  lens-worker:
-    image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
-    environment:
-      LITELLM_URL: http://litellm:4000
-      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
-      LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN}
-      CLICKHOUSE_HOST: clickhouse
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
-      CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
-      AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
-    depends_on: [litellm]
-    networks: [proxy, storage]
-    ports:
-      - "127.0.0.1:${LENS_PORT:-4318}:4318"
-    mem_limit: 2g
-    cpus: 2
-    pids_limit: 64
-    restart: unless-stopped
-    read_only: true
-    tmpfs:
-      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
-    cap_drop: [ALL]
-    security_opt: [no-new-privileges:true]
-
-  db:
-    image: postgres:16
-    environment:
-      POSTGRES_DB: litellm
-      POSTGRES_USER: litellm
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    networks: [database]
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    restart: unless-stopped
-
-  clickhouse:
-    image: clickhouse/clickhouse-server:26.9.6.6
-    environment:
-      CLICKHOUSE_USER: default
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
-      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
-    volumes:
-      - clickhouse_data:/var/lib/clickhouse
-    healthcheck:
-      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    restart: unless-stopped
-    networks: [storage]
-
-networks:
-  proxy:
-  database:
-    internal: true
-  storage:
-    internal: true
-
-volumes:
-  postgres_data:
-  clickhouse_data:

deploy/lens/test_configure.py

@@ -1,52 +0,0 @@
-import tempfile
-import unittest
-from pathlib import Path
-
-from configure import SECRET_NAMES, configure
-
-
-class ComposeConfigurationTests(unittest.TestCase):
-    def test_restart_and_upgrade_preserve_private_credentials_and_custom_settings(self) -> None:
-        with tempfile.TemporaryDirectory() as directory:
-            path = Path(directory) / ".env"
-            configure(path, "v1.2.3")
-            original = dict(line.split("=", 1) for line in path.read_text().splitlines())
-            self.assertEqual(path.stat().st_mode & 0o777, 0o600)
-            self.assertEqual(len({original[name] for name in SECRET_NAMES}), len(SECRET_NAMES))
-            self.assertTrue(all(len(original[name]) >= 64 for name in SECRET_NAMES))
-            with path.open("a") as output:
-                output.write("LITELLM_LENS_PUBLIC_URL=https://traces.example/prefix\n")
-            configure(path, "1.2.3")
-            configure(path, "v1.2.4-nightly")
-            updated = dict(line.split("=", 1) for line in path.read_text().splitlines())
-            self.assertEqual({name: updated[name] for name in SECRET_NAMES}, {name: original[name] for name in SECRET_NAMES})
-            self.assertEqual(updated["LITELLM_VERSION"], "1.2.4-nightly")
-            self.assertEqual(updated["LITELLM_LENS_PUBLIC_URL"], "https://traces.example/prefix")
-            self.assertEqual(path.stat().st_mode & 0o777, 0o600)
-
-    def test_incomplete_configuration_is_never_replaced_with_new_database_passwords(self) -> None:
-        with tempfile.TemporaryDirectory() as directory:
-            path = Path(directory) / ".env"
-            original = "POSTGRES_PASSWORD=existing\n"
-            path.write_text(original)
-            with self.assertRaisesRegex(ValueError, "incomplete"):
-                configure(path, "1.2.3")
-            self.assertEqual(path.read_text(), original)
-
-    def test_invalid_release_and_symlink_leave_existing_files_untouched(self) -> None:
-        with tempfile.TemporaryDirectory() as directory:
-            path = Path(directory) / ".env"
-            target = Path(directory) / "saved"
-            target.write_text("preserve")
-            path.symlink_to(target)
-            with self.assertRaisesRegex(ValueError, "symlink"):
-                configure(path, "1.2.3")
-            self.assertEqual(target.read_text(), "preserve")
-            path.unlink()
-            with self.assertRaisesRegex(ValueError, "published release"):
-                configure(path, "1.2.3\nPOSTGRES_PASSWORD=replaced")
-            self.assertFalse(path.exists())
-
-
-if __name__ == "__main__":
-    unittest.main()

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -58,6 +58,7 @@ RUN apk add --no-cache \
     python-3.13 \
     python-3.13-dev \
     openssl \
+    libssl3 \
     openssl-dev \
     nodejs \
     npm \
@@ -121,7 +122,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -59,6 +59,7 @@ RUN for i in 1 2 3; do \
         coreutils \
         curl \
         openssl \
+        libssl3 \
         libsndfile \
         nodejs \
         npm && break || sleep 5; \
@@ -133,7 +134,7 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
     done
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 

docker/build_from_pip/Dockerfile.build_from_pip

@@ -30,7 +30,7 @@ RUN uv venv --python python && \
       "google-cloud-aiplatform==1.133.0" \
       "google-genai==1.37.0" \
       "anthropic[vertex]==0.84.0" \
-      "grpcio==1.78.0" \
+      "grpcio==1.84.1" \
       "prometheus-client==0.20.0" \
       "langfuse==2.59.7" \
       "opentelemetry-api==1.28.0" \

docker/docker-compose.tracing.yml

@@ -1,86 +0,0 @@
-name: litellm-tracing
-
-services:
-  litellm:
-    build:
-      context: ..
-      target: runtime
-      args:
-        LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:-}
-    command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
-    environment:
-      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set LITELLM_MASTER_KEY}
-      LITELLM_SALT_KEY: sk-local-tracing-salt-key
-      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
-      STORE_MODEL_IN_DB: "True"
-      LITELLM_LENS_URL: http://lens-worker:4318
-      LITELLM_LENS_PUBLIC_URL: http://localhost:4318
-      LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
-      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
-      LENS_WORKER_IMAGE: ${LENS_WORKER_IMAGE:-}
-    volumes:
-      - ./tracing-config.yaml:/app/tracing-config.yaml:ro
-    ports:
-      - "127.0.0.1:4002:4000"
-    depends_on:
-      db:
-        condition: service_healthy
-
-  lens-worker:
-    build:
-      context: ..
-      dockerfile: deploy/lens/Dockerfile
-      args:
-        LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?set LITELLM_RELEASE_TAG to the source commit}
-    environment:
-      LITELLM_URL: http://litellm:4000
-      LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
-      CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
-      CLICKHOUSE_DATABASE: litellm
-    ports:
-      - "127.0.0.1:4318:4318"
-    read_only: true
-    cap_drop: [ALL]
-    security_opt: [no-new-privileges:true]
-    tmpfs:
-      - /tmp:rw,noexec,nosuid,nodev,size=${LENS_WORKER_TMP_SIZE:-1g},mode=1777
-    mem_limit: 2g
-    cpus: 2
-    pids_limit: 64
-    restart: unless-stopped
-
-  db:
-    image: postgres:16
-    environment:
-      POSTGRES_DB: litellm
-      POSTGRES_USER: litellm
-      POSTGRES_PASSWORD: litellm
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    ports:
-      - "127.0.0.1:15432:5432"
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  clickhouse:
-    image: clickhouse/clickhouse-server:26.9.6.6
-    environment:
-      CLICKHOUSE_USER: default
-      CLICKHOUSE_PASSWORD: local-tracing
-      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
-    volumes:
-      - clickhouse_data:/var/lib/clickhouse
-    ports:
-      - "127.0.0.1:18123:8123"
-    healthcheck:
-      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing", "--query", "SELECT 1"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-
-volumes:
-  postgres_data:
-  clickhouse_data:

docker/tracing-config.yaml

@@ -1,11 +0,0 @@
-model_list:
-  - model_name: gpt-6.1-sol
-    litellm_params:
-      model: openai/gpt-6.1-sol
-      api_key: os.environ/OPENAI_API_KEY
-
-general_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
-  tracing:
-    store:
-      type: lens

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,7 +2,9 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
-from collections.abc import Sequence
+import json
+import re
+from collections.abc import Mapping, Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
@@ -31,7 +33,12 @@ if TYPE_CHECKING:
     from litellm.types.utils import LiteLLMBatch
 
 
+def _error_names_resource(error: Exception, resource_id: str) -> bool:
+    return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
+
+
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
+RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
 
 PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
     "completed",
@@ -57,6 +64,15 @@ class _ManagedObjectRow(Protocol):
     @property
     def created_by(self) -> str | None: ...
 
+    @property
+    def created_at(self) -> datetime: ...
+
+    @property
+    def updated_at(self) -> datetime: ...
+
+    @property
+    def status(self) -> str | None: ...
+
     @property
     def file_object(self) -> object: ...
 
@@ -122,6 +138,11 @@ class CheckBatchCost:
         # the guaranteed-failing primary query on every subsequent cycle.
         self._has_batch_processed_column: bool = True
         self.batch_processed_support_confirmed: bool = False
+        self._poll_cursor: tuple[datetime, str] | None = None
+        self._poll_before: datetime = datetime.now(timezone.utc)
+        self._arrival_cursor: tuple[datetime, str] | None = None
+        self._arrival_before: datetime = self._poll_before
+        self._poll_arrivals: bool = False
 
     @staticmethod
     def _is_missing_batch_processed_column_error(err: Exception) -> bool:
@@ -297,29 +318,9 @@ class CheckBatchCost:
                 f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
             )
 
-        if not self._has_batch_processed_column:
-            return
-
-        # A row already in a terminal status is never rewritten by the sweep above, so
-        # without this it keeps a poll-page slot forever and starves newer batches.
-        retired: Final = await _managed_object_table(self.prisma_client).update_many(
-            where={
-                "file_purpose": "batch",
-                "batch_processed": False,
-                "status": {"in": ["complete", "completed"]},
-                "created_at": {"lt": cutoff},
-            },
-            data={"batch_processed": True},
-        )
-        if retired > 0:
-            verbose_proxy_logger.warning(
-                f"CheckBatchCost: gave up on {retired} completed managed objects older than "
-                f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
-            )
-
     async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]":
         """Query batch jobs without the batch_processed filter (for older schemas)."""
-        return await _managed_object_table(self.prisma_client).find_many(
+        return await self._find_jobs(
             where={
                 "file_purpose": "batch",
                 "status": {
@@ -333,10 +334,59 @@ class CheckBatchCost:
                     ]
                 },
             },
-            take=MAX_OBJECTS_PER_POLL_CYCLE,
-            order={"created_at": "asc"},
         )
 
+    async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[_ManagedObjectRow]:
+        arrivals: Final = self._poll_arrivals
+        page: Final = await self._find_job_page(where, arrivals=arrivals)
+        self._poll_arrivals = not arrivals if page else arrivals
+        if page:
+            return page
+        return await self._find_job_page(where, arrivals=not arrivals)
+
+    async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[_ManagedObjectRow]:
+        cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
+        if cursor is None:
+            if arrivals:
+                self._arrival_before = datetime.now(timezone.utc)
+            else:
+                self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
+        before: Final = self._arrival_before if arrivals else self._poll_before
+        page: Final = await _managed_object_table(self.prisma_client).find_many(
+            where={
+                **where,
+                "created_at": {
+                    "lte": before,
+                    **({"gt": before - RECENT_BATCH_POLL_WINDOW} if arrivals else {}),
+                },
+                **(
+                    {
+                        "OR": [
+                            {"created_at": {"gt": cursor[0]}},
+                            {"created_at": cursor[0], "id": {"gt": cursor[1]}},
+                        ]
+                    }
+                    if cursor is not None
+                    else {}
+                ),
+            },
+            take=MAX_OBJECTS_PER_POLL_CYCLE,
+            order=[{"created_at": "asc"}, {"id": "asc"}],
+        )
+        next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
+        if arrivals:
+            self._arrival_cursor = next_cursor
+        else:
+            self._poll_cursor = next_cursor
+        return page
+
+    def _unprocessed_snapshot(self, job: _ManagedObjectRow) -> Mapping[str, object]:
+        return {
+            "id": job.id,
+            "updated_at": job.updated_at,
+            **({"batch_processed": False} if self._has_batch_processed_column else {}),
+        }
+
     async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None:
         """
         Take a row that can never be costed out of the poll page. Leaving it selectable
@@ -350,8 +400,8 @@ class CheckBatchCost:
             else {"status": "stale_expired"}
         )
         try:
-            await _managed_object_table(self.prisma_client).update(
-                where={"id": job.id},
+            retired: Final = await _managed_object_table(self.prisma_client).update_many(
+                where=self._unprocessed_snapshot(job),
                 data=data,
             )
         except Exception as db_err:
@@ -359,12 +409,14 @@ class CheckBatchCost:
                 f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
             )
             return
+        if retired == 0:
+            return
         verbose_proxy_logger.warning(
             f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool:
+    async def _claim_job_for_costing(self, job: "_ManagedObjectRow", response: "LiteLLMBatch") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -378,10 +430,16 @@ class CheckBatchCost:
         """
         if not self._has_batch_processed_column:
             return True
+        from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
+
         try:
             claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
-                data={"batch_processed": True},
+                data={
+                    "batch_processed": True,
+                    "status": response.status if response.status != "completed" else "complete",
+                    "file_object": postgres_storable_json(response),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -390,7 +448,9 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "_ManagedObjectRow") -> None:
+    async def _release_job_claim(
+        self, job: "_ManagedObjectRow", original_status: str | None, original_file_object: object
+    ) -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -402,7 +462,11 @@ class CheckBatchCost:
         try:
             await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
-                data={"batch_processed": False},
+                data={
+                    "batch_processed": False,
+                    "status": original_status,
+                    "file_object": json.dumps(original_file_object),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -436,13 +500,13 @@ class CheckBatchCost:
 
         from litellm.exceptions import NotFoundError
 
-        return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
 
     def _batch_deployment_exists(self, model_id: str) -> bool:
         """A 404 only proves the batch is gone when it came from the batch's own
         deployment. Once that deployment leaves the router, default fallbacks can
         silently send the retrieve to a provider that never saw the batch, so its
-        404 must not retire the row; the staleness sweep bounds it instead."""
+        404 must not retire a potentially recoverable row."""
         return self.llm_router.get_deployment(model_id=model_id) is not None
 
     @staticmethod
@@ -457,7 +521,7 @@ class CheckBatchCost:
 
         if not output_file_id:
             return False
-        return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
 
     async def _finalize_unbilled_terminal_job(
         self, job: "_ManagedObjectRow", response: "LiteLLMBatch"
@@ -484,10 +548,12 @@ class CheckBatchCost:
                 "file_object": response.model_dump_json(),
                 **({"batch_processed": True} if self._has_batch_processed_column else {}),
             }
-            await _managed_object_table(self.prisma_client).update(
-                where={"id": job.id},
+            finalized: Final = await _managed_object_table(self.prisma_client).update_many(
+                where=self._unprocessed_snapshot(job),
                 data=update_data,
             )
+            if finalized == 0:
+                return
             verbose_proxy_logger.info(
                 f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
             )
@@ -699,8 +765,6 @@ class CheckBatchCost:
 
     @staticmethod
     def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]:
-        import json
-
         from litellm.types.utils import LiteLLMBatch
 
         file_object: Final = job.file_object
@@ -743,7 +807,11 @@ class CheckBatchCost:
         from litellm.files.types import FileContentCallOptions
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
+        from litellm.litellm_core_utils.litellm_logging import (
+            deployment_pricing_model_info,
+            mask_api_base_credentials,
+            resolve_vertex_location_for_cost,
+        )
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -873,6 +941,12 @@ class CheckBatchCost:
             custom_llm_provider=batch_file_provider,
             model_name=model_name,
             model_info=deployment_model_info,
+            vertex_location=resolve_vertex_location_for_cost(
+                custom_llm_provider=llm_provider,
+                litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
+                optional_params=None,
+                model=model_name,
+            ),
         )
         error_file_failed_requests: Final = await count_error_file_failed_requests(
             response,
@@ -890,8 +964,9 @@ class CheckBatchCost:
                 failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
+        logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
         logging_obj = LiteLLMLogging(
-            model=batch_result.models[0],
+            model=logging_model,
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -922,27 +997,35 @@ class CheckBatchCost:
             custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
-        if not await self._claim_job_for_costing(job):
+        original_status: Final = job.status
+        original_file_object: Final = job.file_object
+        if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
             verbose_proxy_logger.info(
                 f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
                 "in this window, so its cost is already being tracked there"
             )
             return None
 
-        try:
-            await logging_obj.async_success_handler(
-                result=response,
-                batch_cost=batch_result.cost,
-                batch_usage=batch_result.usage,
-                batch_models=batch_result.models,
-                batch_successful_requests=batch_result.successful_requests,
-                batch_failed_requests=batch_result.failed_requests,
-                batch_prompt_cost=batch_result.prompt_cost,
-                batch_completion_cost=batch_result.completion_cost,
-            )
-        except Exception:
-            await self._release_job_claim(job)
-            raise
+        from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
+
+        with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
+            try:
+                await logging_obj.async_success_handler(
+                    result=response,
+                    batch_cost=batch_result.cost,
+                    batch_usage=batch_result.usage,
+                    batch_models=batch_result.models,
+                    batch_successful_requests=batch_result.successful_requests,
+                    batch_failed_requests=batch_result.failed_requests,
+                    batch_prompt_cost=batch_result.prompt_cost,
+                    batch_completion_cost=batch_result.completion_cost,
+                )
+            finally:
+                if accounting.retryable_failure:
+                    await self._release_job_claim(job, original_status, original_file_object)
+
+        if accounting.retryable_failure:
+            return None
 
         # Record batch duration (completed_at - created_at)
         if prom_logger and response.completed_at and response.created_at:
@@ -990,7 +1073,7 @@ class CheckBatchCost:
                 # endpoint may transition a batch to "complete" before
                 # CheckBatchCost runs.  The batch_processed=False filter
                 # already prevents reprocessing finished batches.
-                jobs = await _managed_object_table(self.prisma_client).find_many(
+                jobs = await self._find_jobs(
                     where={
                         "file_purpose": "batch",
                         "batch_processed": False,
@@ -1003,8 +1086,6 @@ class CheckBatchCost:
                             ]
                         },
                     },
-                    take=MAX_OBJECTS_PER_POLL_CYCLE,
-                    order={"created_at": "asc"},
                 )
                 self.batch_processed_support_confirmed = True
             except Exception as query_err:
@@ -1074,7 +1155,7 @@ class CheckBatchCost:
                         continue
                     verbose_proxy_logger.error(
                         f"CheckBatchCost: failed to track cost for batch {batch_id} "
-                        f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
+                        f"(job {job.id}): {tracking_err}"
                     )
                     self._record_error(prom_logger, "cost_tracking_error")
                     continue
@@ -1084,22 +1165,19 @@ class CheckBatchCost:
                 # Track this job for the final metrics summary
                 processed_models.append(tracked)
 
-                # mark the job as complete
-                try:
-                    update_data: dict = {
-                        "status": response.status if response.status != "completed" else "complete",
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await _managed_object_table(self.prisma_client).update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                    )
+                if not self._has_batch_processed_column:
+                    try:
+                        await _managed_object_table(self.prisma_client).update(
+                            where={"id": job.id},
+                            data={
+                                "status": response.status if response.status != "completed" else "complete",
+                                "file_object": response.model_dump_json(),
+                            },
+                        )
+                    except Exception as db_err:
+                        verbose_proxy_logger.error(
+                            f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
+                        )
 
             elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
                 from litellm.proxy.openai_files_endpoints.common_utils import (

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -24,7 +24,7 @@ from uuid import NAMESPACE_URL, uuid5
 
 import httpx
 from fastapi import HTTPException
-from pydantic import ValidationError
+from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, Unpack
 
 import litellm
@@ -347,6 +347,31 @@ def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _Manag
     return prisma_client.db.litellm_managedobjecttable
 
 
+_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+_POSTGRES_UNSTORABLE_CHARACTER: Final = "\x00"
+
+
+def _storable_text(text: str) -> str:
+    return text.replace(_POSTGRES_UNSTORABLE_CHARACTER, "")
+
+
+def _storable_json_value(value: JsonValue) -> JsonValue:
+    if isinstance(value, str):
+        return _storable_text(value)
+    if isinstance(value, list):
+        return [_storable_json_value(item) for item in value]
+    if isinstance(value, dict):
+        return {_storable_text(key): _storable_json_value(item) for key, item in value.items()}
+    return value
+
+
+def postgres_storable_json(file_object: BaseModel) -> str:
+    dumped: Final = file_object.model_dump_json()
+    if "\\u0000" not in dumped:
+        return dumped
+    return _JSON_VALUE_ADAPTER.dump_json(_storable_json_value(_JSON_VALUE_ADAPTER.validate_json(dumped))).decode()
+
+
 def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]:
     hidden_params: Final = cast(  # cast-ok: _hidden_params is an untyped attribute the upload path sets
         "Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {}

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.76"
+version = "0.1.76.post1"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.76"
+version = "0.1.76.post1"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
 ARG PGBOUNCER_VERSION=1.26.0
@@ -32,7 +32,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -95,7 +95,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
+      apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -110,7 +110,12 @@ ENV HOME=/home/nonroot \
     PYTHONUNBUFFERED=1 \
     PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
-COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
+COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
+COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
+COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
+COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
+COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma
 COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 

helm/litellm-helm/templates/_helpers.tpl

@@ -322,24 +322,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
   value: {{ .Values.collector.drainTimeoutSeconds | quote }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.image" -}}
-{{- if .Values.lensWorker.image.digest -}}
-{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}}
-{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}}
-{{- end -}}
-{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}}
-{{- else -}}
-{{- $backendTag := .Values.image.tag | default .Chart.AppVersion -}}
-{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
-{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
-{{- $repository := .Values.lensWorker.image.repository -}}
-{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}}
-{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}}
-{{- end -}}
-{{- printf "%s:%s" $repository $tag -}}
-{{- end -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -366,35 +348,8 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.labels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
-{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
+{{- define "litellm.lensWorkerRemoved" -}}
+{{- if and .Values.lensWorker .Values.lensWorker.enabled }}
+{{- fail "lensWorker was removed; this chart no longer bundles Lens" }}
+{{- end }}
 {{- end -}}

helm/litellm-helm/templates/deployment.yaml

@@ -56,17 +56,7 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
-            {{- if .Values.lensWorker.enabled }}
-            - name: LITELLM_LENS_URL
-              value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
-            - name: LITELLM_LENS_PUBLIC_URL
-              value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
-            - name: LITELLM_LENS_SERVICE_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-                  key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
-            {{- end }}
+            {{- include "litellm.lensWorkerRemoved" . }}
             {{- include "litellm.proxyEnv" . | nindent 12 }}
             {{- if .Values.liteadmin.enabled }}
             - name: LITELLM_ADMIN_AGENT_URL

helm/litellm-helm/templates/ingress.yaml

@@ -44,20 +44,6 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          {{- if $.Values.lensWorker.enabled }}
-          - path: /lens-ingest
-            pathType: Prefix
-            backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}-lens-worker
-                port:
-                  number: {{ $.Values.lensWorker.service.port }}
-              {{- else }}
-              serviceName: {{ $fullName }}-lens-worker
-              servicePort: {{ $.Values.lensWorker.service.port }}
-              {{- end }}
-          {{- end }}
           {{- range .paths }}
           - path: {{ .path }}
             {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}

helm/litellm-helm/templates/lens/clickhouse.yaml

@@ -1,98 +0,0 @@
-{{- if include "litellm.lensWorker.bundledClickhouse" . }}
-{{- $name := include "litellm.lensWorker.clickhouseName" . }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $name }}
-spec:
-  clusterIP: None
-  selector:
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: lens-clickhouse
-  ports:
-    - name: http
-      port: 8123
-      targetPort: http
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: {{ $name }}
-spec:
-  serviceName: {{ $name }}
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: {{ .Release.Name }}
-      app.kubernetes.io/component: lens-clickhouse
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: {{ .Release.Name }}
-        app.kubernetes.io/component: lens-clickhouse
-    spec:
-      automountServiceAccountToken: false
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 101
-        runAsGroup: 101
-        fsGroup: 101
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: clickhouse
-          image: {{ .Values.lensWorker.clickhouse.image | quote }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: [ALL]
-          env:
-            - name: CLICKHOUSE_USER
-              value: default
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $name }}
-                  key: password
-            - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
-              value: "1"
-          ports:
-            - name: http
-              containerPort: 8123
-          startupProbe:
-            httpGet:
-              path: /ping
-              port: http
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-          readinessProbe:
-            httpGet:
-              path: /ping
-              port: http
-          livenessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            timeoutSeconds: 3
-          resources:
-            {{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
-          volumeMounts:
-            - name: data
-              mountPath: /var/lib/clickhouse
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: [ReadWriteOnce]
-        {{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
-        storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
-        {{- end }}
-        resources:
-          requests:
-            storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
-{{- end }}

helm/litellm-helm/templates/lens/deployment.yaml

@@ -1,109 +0,0 @@
-{{- if .Values.lensWorker.enabled }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  labels:
-    {{- include "litellm.lensWorker.labels" . | nindent 4 }}
-    app.kubernetes.io/component: lens-worker
-spec:
-  replicas: {{ .Values.lensWorker.replicaCount }}
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: {{ .Release.Name }}
-      app.kubernetes.io/component: lens-worker
-  template:
-    metadata:
-      labels:
-        {{- include "litellm.lensWorker.labels" . | nindent 8 }}
-        app.kubernetes.io/component: lens-worker
-    spec:
-      automountServiceAccountToken: false
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: lens-worker
-          image: {{ include "litellm.lensWorker.image" . | quote }}
-          imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: [ALL]
-          env:
-            - name: LITELLM_URL
-              value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.fullname" .) .Values.service.port) | quote }}
-            - name: LITELLM_LENS_SERVICE_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-                  key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
-            {{- if include "litellm.lensWorker.bundledClickhouse" . }}
-            - name: CLICKHOUSE_HOST
-              value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
-                  key: password
-            {{- else }}
-            - name: CLICKHOUSE_URL
-              valueFrom:
-                secretKeyRef:
-                  name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
-                  key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
-            {{- end }}
-            - name: CLICKHOUSE_DATABASE
-              value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
-            - name: AGENT_TRACING_RETENTION_DAYS
-              value: {{ .Values.lensWorker.retentionDays | quote }}
-            {{- if .Values.lensWorker.tokenSecret.name }}
-            - name: LENS_WORKER_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.lensWorker.tokenSecret.name | quote }}
-                  key: {{ .Values.lensWorker.tokenSecret.key | quote }}
-            {{- end }}
-          ports:
-            - name: otlp
-              containerPort: 4318
-          livenessProbe:
-            httpGet:
-              path: /health/live
-              port: otlp
-          readinessProbe:
-            httpGet:
-              path: /health/ready
-              port: otlp
-          resources:
-            {{- toYaml .Values.lensWorker.resources | nindent 12 }}
-          volumeMounts:
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: tmp
-          emptyDir:
-            medium: Memory
-            sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
-      {{- with .Values.lensWorker.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.lensWorker.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.lensWorker.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

helm/litellm-helm/templates/lens/ingress.yaml

@@ -1,29 +0,0 @@
-{{- if and .Values.lensWorker.enabled .Values.lensWorker.ingress.enabled }}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  {{- with .Values.lensWorker.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- with .Values.lensWorker.ingress.className }}
-  ingressClassName: {{ . | quote }}
-  {{- end }}
-  {{- with .Values.lensWorker.ingress.tls }}
-  tls:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    - host: {{ required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host | quote }}
-      http:
-        paths:
-          - path: /v1/
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ include "litellm.fullname" . }}-lens-worker
-                port:
-                  name: otlp
-{{- end }}

helm/litellm-helm/templates/lens/secrets.yaml

@@ -1,27 +0,0 @@
-{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
-{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
-{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}
-  annotations:
-    helm.sh/resource-policy: keep
-type: Opaque
-data:
-  {{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
-{{- end }}
-{{- if include "litellm.lensWorker.bundledClickhouse" . }}
-{{- $name := include "litellm.lensWorker.clickhouseName" . }}
-{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}
-  annotations:
-    helm.sh/resource-policy: keep
-type: Opaque
-data:
-  password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
-{{- end }}

helm/litellm-helm/templates/lens/service.yaml

@@ -1,18 +0,0 @@
-{{- if .Values.lensWorker.enabled }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  {{- with .Values.lensWorker.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  selector:
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: lens-worker
-  ports:
-    - name: otlp
-      port: {{ .Values.lensWorker.service.port }}
-      targetPort: otlp
-{{- end }}

helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,84 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - deployment.yaml
-  - configmap-litellm.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest

helm/litellm-helm/tests/lens_saved_secrets_tests.yaml

@@ -1,45 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

helm/litellm-helm/tests/lens_service_tests.yaml

@@ -1,167 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects deployment.yaml to the shared Lens service
-  template: deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef:
-            name: lens-service
-            key: service-token
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.hosts:
-    - host: gateway.example
-      paths:
-      - path: /
-        pathType: Prefix
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in deployment.yaml
-  template: deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference

helm/litellm-helm/tests/lens_setup_tests.yaml

@@ -1,160 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates both private credentials for a new installation
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 2
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: lensWorker.clickhouseSecret.name is required
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60

helm/litellm-helm/values.yaml

@@ -653,54 +653,3 @@ serviceMonitor:
     matchNames: []
     # - test-namespace
 
-lensWorker:
-  enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/litellm-lens-worker
-    tag: ""
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
-    name: ""
-    key: token
-  serviceTokenSecret:
-    name: ""
-    key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

helm/litellm/templates/_helpers.tpl

@@ -471,24 +471,6 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
-{{- define "litellm.lensWorker.image" -}}
-{{- if .Values.lensWorker.image.digest -}}
-{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}}
-{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}}
-{{- end -}}
-{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}}
-{{- else -}}
-{{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}}
-{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
-{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
-{{- $repository := .Values.lensWorker.image.repository -}}
-{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}}
-{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}}
-{{- end -}}
-{{- printf "%s:%s" $repository $tag -}}
-{{- end -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -515,48 +497,8 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensConnectionEnv" -}}
-{{- if .Values.lensWorker.enabled }}
-- name: LITELLM_LENS_URL
-  value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
-- name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
-- name: LITELLM_LENS_SERVICE_TOKEN
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+{{- define "litellm.lensWorkerRemoved" -}}
+{{- if and .Values.lensWorker .Values.lensWorker.enabled }}
+{{- fail "lensWorker was removed; this chart no longer bundles Lens" }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}

helm/litellm/templates/backend/deployment.yaml

@@ -57,9 +57,7 @@ spec:
               containerPort: 4001
               protocol: TCP
           env:
-            {{- include "litellm.lensConnectionEnv" . | nindent 12 }}
-            - name: LENS_WORKER_IMAGE
-              value: {{ include "litellm.lensWorker.image" . | quote }}
+            {{- include "litellm.lensWorkerRemoved" . }}
             {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
             {{- if .Values.gateway.config.create }}
             - name: CONFIG_FILE_PATH

helm/litellm/templates/gateway/deployment.yaml

@@ -55,7 +55,7 @@ spec:
               containerPort: 4000
               protocol: TCP
           env:
-            {{- include "litellm.lensConnectionEnv" . | nindent 12 }}
+            {{- include "litellm.lensWorkerRemoved" . }}
             {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
             {{- if .Values.gateway.config.create }}
             - name: CONFIG_FILE_PATH

helm/litellm/templates/ingress.yaml

@@ -156,16 +156,6 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
-          {{- if .Values.lensWorker.enabled }}
-          {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
-          - path: /lens-ingest
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ include "litellm.fullname" . }}-lens-worker
-                port:
-                  number: {{ .Values.lensWorker.service.port }}
-          {{- end }}
           {{- /*
             --- Operator-supplied extra paths (ingress.extraPaths) ---
             Rendered after every built-in path so an entry can never take

helm/litellm/templates/lens/clickhouse.yaml

@@ -1,98 +0,0 @@
-{{- if include "litellm.lensWorker.bundledClickhouse" . }}
-{{- $name := include "litellm.lensWorker.clickhouseName" . }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $name }}
-spec:
-  clusterIP: None
-  selector:
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: lens-clickhouse
-  ports:
-    - name: http
-      port: 8123
-      targetPort: http
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: {{ $name }}
-spec:
-  serviceName: {{ $name }}
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: {{ .Release.Name }}
-      app.kubernetes.io/component: lens-clickhouse
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: {{ .Release.Name }}
-        app.kubernetes.io/component: lens-clickhouse
-    spec:
-      automountServiceAccountToken: false
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 101
-        runAsGroup: 101
-        fsGroup: 101
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: clickhouse
-          image: {{ .Values.lensWorker.clickhouse.image | quote }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: [ALL]
-          env:
-            - name: CLICKHOUSE_USER
-              value: default
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $name }}
-                  key: password
-            - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
-              value: "1"
-          ports:
-            - name: http
-              containerPort: 8123
-          startupProbe:
-            httpGet:
-              path: /ping
-              port: http
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-          readinessProbe:
-            httpGet:
-              path: /ping
-              port: http
-          livenessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            timeoutSeconds: 3
-          resources:
-            {{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
-          volumeMounts:
-            - name: data
-              mountPath: /var/lib/clickhouse
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: [ReadWriteOnce]
-        {{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
-        storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
-        {{- end }}
-        resources:
-          requests:
-            storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
-{{- end }}

helm/litellm/templates/lens/deployment.yaml

@@ -1,109 +0,0 @@
-{{- if .Values.lensWorker.enabled }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  labels:
-    {{- include "litellm.lensWorker.labels" . | nindent 4 }}
-    app.kubernetes.io/component: lens-worker
-spec:
-  replicas: {{ .Values.lensWorker.replicaCount }}
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: {{ .Release.Name }}
-      app.kubernetes.io/component: lens-worker
-  template:
-    metadata:
-      labels:
-        {{- include "litellm.lensWorker.labels" . | nindent 8 }}
-        app.kubernetes.io/component: lens-worker
-    spec:
-      automountServiceAccountToken: false
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: lens-worker
-          image: {{ include "litellm.lensWorker.image" . | quote }}
-          imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: [ALL]
-          env:
-            - name: LITELLM_URL
-              value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
-            - name: LITELLM_LENS_SERVICE_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-                  key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
-            {{- if include "litellm.lensWorker.bundledClickhouse" . }}
-            - name: CLICKHOUSE_HOST
-              value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
-                  key: password
-            {{- else }}
-            - name: CLICKHOUSE_URL
-              valueFrom:
-                secretKeyRef:
-                  name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
-                  key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
-            {{- end }}
-            - name: CLICKHOUSE_DATABASE
-              value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
-            - name: AGENT_TRACING_RETENTION_DAYS
-              value: {{ .Values.lensWorker.retentionDays | quote }}
-            {{- if .Values.lensWorker.tokenSecret.name }}
-            - name: LENS_WORKER_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.lensWorker.tokenSecret.name | quote }}
-                  key: {{ .Values.lensWorker.tokenSecret.key | quote }}
-            {{- end }}
-          ports:
-            - name: otlp
-              containerPort: 4318
-          livenessProbe:
-            httpGet:
-              path: /health/live
-              port: otlp
-          readinessProbe:
-            httpGet:
-              path: /health/ready
-              port: otlp
-          resources:
-            {{- toYaml .Values.lensWorker.resources | nindent 12 }}
-          volumeMounts:
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: tmp
-          emptyDir:
-            medium: Memory
-            sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
-      {{- with .Values.lensWorker.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.lensWorker.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.lensWorker.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

helm/litellm/templates/lens/ingress.yaml

@@ -1,29 +0,0 @@
-{{- if and .Values.lensWorker.enabled .Values.lensWorker.ingress.enabled }}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  {{- with .Values.lensWorker.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- with .Values.lensWorker.ingress.className }}
-  ingressClassName: {{ . | quote }}
-  {{- end }}
-  {{- with .Values.lensWorker.ingress.tls }}
-  tls:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  rules:
-    - host: {{ required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host | quote }}
-      http:
-        paths:
-          - path: /v1/
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ include "litellm.fullname" . }}-lens-worker
-                port:
-                  name: otlp
-{{- end }}

helm/litellm/templates/lens/secrets.yaml

@@ -1,27 +0,0 @@
-{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
-{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
-{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}
-  annotations:
-    helm.sh/resource-policy: keep
-type: Opaque
-data:
-  {{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
-{{- end }}
-{{- if include "litellm.lensWorker.bundledClickhouse" . }}
-{{- $name := include "litellm.lensWorker.clickhouseName" . }}
-{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}
-  annotations:
-    helm.sh/resource-policy: keep
-type: Opaque
-data:
-  password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
-{{- end }}

helm/litellm/templates/lens/service.yaml

@@ -1,18 +0,0 @@
-{{- if .Values.lensWorker.enabled }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "litellm.fullname" . }}-lens-worker
-  {{- with .Values.lensWorker.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  selector:
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/component: lens-worker
-  ports:
-    - name: otlp
-      port: {{ .Values.lensWorker.service.port }}
-      targetPort: otlp
-{{- end }}

helm/litellm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,86 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - gateway/deployment.yaml
-  - gateway/configmap.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.host: gateway.example
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_saved_secrets_tests.yaml

@@ -1,47 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_service_tests.yaml

@@ -1,196 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- gateway/configmap.yaml
-- gateway/deployment.yaml
-- backend/deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects gateway/deployment.yaml to the shared Lens service
-  template: gateway/deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef:
-            name: lens-service
-            key: service-token
-- it: connects backend/deployment.yaml to the shared Lens service
-  template: backend/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef:
-            name: lens-service
-            key: service-token
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.host: gateway.example
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: gateway/deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in gateway/deployment.yaml
-  template: gateway/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: omits Lens connection settings when disabled in backend/deployment.yaml
-  template: backend/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: gateway/deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference
-values:
-- ./values/required.yaml

helm/litellm/tests/lens_setup_tests.yaml

@@ -1,162 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates both private credentials for a new installation
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 2
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: lensWorker.clickhouseSecret.name is required
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_worker_tests.yaml

@@ -1,197 +0,0 @@
-suite: Lens worker release and credentials
-templates:
-  - lens/deployment.yaml
-  - backend/deployment.yaml
-  - gateway/configmap.yaml
-values:
-  - ./values/required.yaml
-tests:
-  - it: installs the development package for a source commit
-    template: lens/deployment.yaml
-    set:
-      backend.image.tag: sha-0123456789abcdef
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef
-  - it: advertises the development package for standalone source workers
-    template: backend/deployment.yaml
-    set:
-      backend.image.tag: sha-0123456789abcdef
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_WORKER_IMAGE
-            value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef
-  - it: preserves an explicit private source image repository
-    template: lens/deployment.yaml
-    set:
-      backend.image.tag: sha-0123456789abcdef
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-      lensWorker.image.repository: registry.example/lens-worker
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens-worker:sha-0123456789abcdef
-  - it: pins the worker to its approved digest even when its tag changes
-    template: lens/deployment.yaml
-    set:
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-      lensWorker.image.tag: replaced-release
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: advertises the approved digest to standalone installers
-    template: backend/deployment.yaml
-    set:
-      lensWorker.image.tag: replaced-release
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_WORKER_IMAGE
-            value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: refuses a malformed digest instead of falling back to the tag
-    template: backend/deployment.yaml
-    set:
-      lensWorker.image.digest: sha256:invalid
-    asserts:
-      - failedTemplate:
-          errorMessage: lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters
-  - it: keeps the worker opt in
-    template: lens/deployment.yaml
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: uses the managed service secret when none is supplied
-    template: lens/deployment.yaml
-    set:
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_SERVICE_TOKEN
-            valueFrom:
-              secretKeyRef:
-                name: RELEASE-NAME-litellm-lens-service
-                key: service-token
-  - it: uses the chart release and a secret without granting Kubernetes access
-    template: lens/deployment.yaml
-    chart:
-      appVersion: v1.2.3
-    set:
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3
-      - equal:
-          path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
-          value:
-            name: lens-service
-            key: service-token
-      - equal:
-          path: spec.template.spec.automountServiceAccountToken
-          value: false
-      - equal:
-          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
-          value: true
-      - equal:
-          path: spec.template.spec.volumes[0].emptyDir
-          value:
-            medium: Memory
-            sizeLimit: 1Gi
-  - it: advertises the same private dev image to standalone installers
-    template: backend/deployment.yaml
-    set:
-      lensWorker.image.repository: registry.example/lens-worker
-      lensWorker.image.tag: branch-main-1234567
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_WORKER_IMAGE
-            value: registry.example/lens-worker:branch-main-1234567
-  - it: supports an external gateway and a registry override
-    template: lens/deployment.yaml
-    set:
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-      lensWorker.url: https://gateway.example/proxy
-      lensWorker.image.repository: registry.example/lens-worker
-      lensWorker.image.tag: branch-main-1234567
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens-worker:branch-main-1234567
-      - equal:
-          path: spec.template.spec.containers[0].env[0].value
-          value: https://gateway.example/proxy
-  - it: prefixes a numeric chart release with v
-    template: lens/deployment.yaml
-    chart:
-      appVersion: 1.2.3-rc.4
-    set:
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-rc.4
-  - it: follows a backend image override when no worker tag is set
-    template: lens/deployment.yaml
-    set:
-      backend.image.tag: branch-main-1234567
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker:branch-main-1234567
-  - it: recommends the overridden backend release for standalone installers
-    template: backend/deployment.yaml
-    set:
-      backend.image.tag: v1.2.3-dev.4
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_WORKER_IMAGE
-            value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4
-  - it: normalizes a numeric backend tag to the published worker tag
-    template: lens/deployment.yaml
-    set:
-      backend.image.tag: 1.2.3-dev.4
-      lensWorker.enabled: true
-      lensWorker.publicUrl: https://traces.example
-      lensWorker.serviceTokenSecret.name: lens-service
-      lensWorker.clickhouseSecret.name: lens-storage
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4

helm/litellm/values.yaml

@@ -630,54 +630,3 @@ ui:
   # Same shape as gateway.topologySpreadConstraints.
   topologySpreadConstraints: []
 
-lensWorker:
-  enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/litellm-lens-worker
-    tag: ""
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
-    name: ""
-    key: token
-  serviceTokenSecret:
-    name: ""
-    key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

litellm-rust/Cargo.lock

@@ -8448,18 +8448,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
 dependencies = [
  "rustls-pki-types",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
+checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
 dependencies = [
  "rustls-pki-types",
 ]

litellm-rust/Cargo.toml

@@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"
 percent-encoding = "2.3"
-webpki-roots = "1"
+webpki-roots = "1.0.10"
 time = { version = "0.3.53", features = ["parsing"] }
 criterion = "0.8.2"
 fancy-regex = "0.19.2"

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -110,7 +110,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        merge_beta_headers(
+            update_headers_with_anthropic_beta(headers, request),
+            safeguards_betas(&request.params),
+        )
     }
 }
 
@@ -198,6 +201,20 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
             .is_some_and(|config| config.format.is_some())
 }
 
+fn safeguards_betas(params: &MessagesOptionalParams) -> BetaSet {
+    carries_safeguards(params)
+        .then_some(AnthropicBeta::DangerousToolUse20260903)
+        .into_iter()
+        .collect()
+}
+
+fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
+    params
+        .extra
+        .get("safeguards")
+        .is_some_and(|value| !value.is_null())
+}
+
 fn messages_carry_output_config(messages: &[Message]) -> bool {
     messages
         .iter()
@@ -1052,6 +1069,12 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
         &[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
     )]
+    #[case::safeguards_add_the_dangerous_tool_use_beta(
+        &[("anthropic-beta", "claude-code-20250219")],
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")],
+    )]
+    #[case::null_safeguards_add_no_beta(&[("x-api-key", "k")], json!({"safeguards": null}), &[("x-api-key", "k")])]
     fn request_headers_merge_the_feature_betas(
         #[case] input: &[(&str, &str)],
         #[case] fields: Value,

litellm-rust/crates/model-catalog/src/capabilities.rs

@@ -9,6 +9,7 @@ pub enum Mode {
     AudioTranscription,
     Chat,
     Completion,
+    Decisions,
     Embedding,
     Evaluation,
     Guardrail,

litellm-rust/crates/model-catalog/tests/catalog.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 
-use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance};
+use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance};
 use rstest::{fixture, rstest};
 use serde_json::json;
 
@@ -42,6 +42,20 @@ fn fixture_catalog() -> Catalog {
     .unwrap()
 }
 
+#[rstest]
+#[case::decisions("decisions", Mode::Decisions)]
+#[case::legacy_evaluation("evaluation", Mode::Evaluation)]
+fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) {
+    let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string();
+    let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap();
+
+    assert_eq!(
+        catalog.lookup("Test").unwrap().entry.info().mode,
+        Some(expected)
+    );
+    assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode));
+}
+
 #[rstest]
 #[ignore]
 fn preserves_fields_and_metadata(fixture_catalog: Catalog) {

litellm/__init__.py

@@ -566,6 +566,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
 
 model_cost = get_model_cost_map(url=model_cost_map_url)
 cost_discount_config: Dict[str, float] = {}  # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
+apply_search_free_tier: bool = False
 cost_margin_config: Dict[
     str, Union[float, Dict[str, float]]
 ] = {}  # Provider-specific or global cost margins. Examples:
@@ -1478,6 +1479,7 @@ from .rust_bridge import rust
 from .rag.main import *
 from .sandbox.main import *
 from .decisions.main import *
+from .systemone.main import *
 from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
 from .search.main import *
 from .realtime_api.main import (
@@ -1681,9 +1683,18 @@ if TYPE_CHECKING:
     from .llms.strands_decider.decisions.transformation import (
         StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
     )
+    from .llms.databricks.decisions.transformation import (
+        DatabricksDecisionsConfig as DatabricksDecisionsConfig,
+    )
+    from .llms.hosted_vllm.decisions.transformation import (
+        HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig,
+    )
     from .llms.openai.decisions.transformation import (
         OpenAIDecisionsConfig as OpenAIDecisionsConfig,
     )
+    from .llms.azure_ai.decisions.transformation import (
+        AzureAIDecisionsConfig as AzureAIDecisionsConfig,
+    )
     from .llms.nvidia_nim.rerank.transformation import (
         NvidiaNimRerankConfig as NvidiaNimRerankConfig,
     )
@@ -2251,6 +2262,8 @@ if TYPE_CHECKING:
     supports_parallel_function_calling: Callable[..., bool]
     supports_vision: Callable[..., bool]
     supports_audio_input: Callable[..., bool]
+    supports_video_input: Callable[..., bool]
+    supports_box_questions: Callable[..., bool]
     supports_audio_output: Callable[..., bool]
     supports_system_messages: Callable[..., bool]
     supports_reasoning: Callable[..., bool]

litellm/_lazy_imports_registry.py

@@ -36,6 +36,8 @@ UTILS_NAMES: Final = (
     "supports_parallel_function_calling",
     "supports_vision",
     "supports_audio_input",
+    "supports_video_input",
+    "supports_box_questions",
     "supports_audio_output",
     "supports_system_messages",
     "supports_reasoning",
@@ -162,7 +164,10 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterDecisionsConfig",
     "CloudflareDecisionsConfig",
     "StrandsDeciderDecisionsConfig",
+    "DatabricksDecisionsConfig",
+    "HostedVLLMDecisionsConfig",
     "OpenAIDecisionsConfig",
+    "AzureAIDecisionsConfig",
     "NvidiaNimRerankConfig",
     "NvidiaNimRankingConfig",
     "VertexAIRerankConfig",
@@ -477,6 +482,8 @@ _UTILS_IMPORT_MAP: Final = {
     ),
     "supports_vision": (".utils", "supports_vision"),
     "supports_audio_input": (".utils", "supports_audio_input"),
+    "supports_video_input": (".utils", "supports_video_input"),
+    "supports_box_questions": (".utils", "supports_box_questions"),
     "supports_audio_output": (".utils", "supports_audio_output"),
     "supports_system_messages": (".utils", "supports_system_messages"),
     "supports_reasoning": (".utils", "supports_reasoning"),
@@ -723,7 +730,13 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.strands_decider.decisions.transformation",
         "StrandsDeciderDecisionsConfig",
     ),
+    "DatabricksDecisionsConfig": (".llms.databricks.decisions.transformation", "DatabricksDecisionsConfig"),
+    "HostedVLLMDecisionsConfig": (
+        ".llms.hosted_vllm.decisions.transformation",
+        "HostedVLLMDecisionsConfig",
+    ),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
+    "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
     "NvidiaNimRerankConfig": (
         ".llms.nvidia_nim.rerank.transformation",
         "NvidiaNimRerankConfig",

litellm/anthropic_beta_headers_config.json

@@ -229,7 +229,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
-    "dangerous-tool-use-2026-09-03": null,
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",

litellm/anthropic_beta_headers_manager.py

@@ -22,17 +22,28 @@ Environment Variables:
 - LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
 """
 
+import asyncio
 import json
 import os
+import threading
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from functools import cache
 from importlib.resources import files
-from typing import Final
+from typing import Final, TypeAlias
 
 import httpx
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 
 # Cache for the loaded configuration
 _BETA_HEADERS_CONFIG: dict | None = None
+_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
+_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
+
+BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
+_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 
 
 class GetAnthropicBetaHeadersConfig:
@@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
     if _BETA_HEADERS_CONFIG is not None:
         return _BETA_HEADERS_CONFIG
 
-    # Get the URL from environment or use default
-    from litellm import anthropic_beta_headers_url
+    with _BETA_HEADERS_CONFIG_LOCK:
+        if _BETA_HEADERS_CONFIG is not None:
+            return _BETA_HEADERS_CONFIG
 
-    _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
-    verbose_logger.debug("Loaded and cached beta headers config")
+        # Get the URL from environment or use default
+        from litellm import anthropic_beta_headers_url
 
-    return _BETA_HEADERS_CONFIG
+        _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
+        verbose_logger.debug("Loaded and cached beta headers config")
+
+        return _BETA_HEADERS_CONFIG
+
+
+async def aload_beta_headers_config() -> None:
+    if _BETA_HEADERS_CONFIG is None:
+        await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
+
+
+@cache
+def _bundled_beta_headers_config() -> BetaHeadersConfig:
+    return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
 
 
 def reload_beta_headers_config() -> dict:
@@ -288,6 +313,14 @@ def is_beta_header_supported(
     return beta_header in provider_mapping and provider_mapping[beta_header] is not None
 
 
+def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    provider_mapping: Final = config.get(canonical_provider, {})
+    return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
+
+
 def get_provider_beta_header(
     anthropic_beta_header: str,
     provider: str,

litellm/batches/batch_utils.py

@@ -74,6 +74,7 @@ async def calculate_batch_cost_and_usage(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Calculate the cost and usage of a batch.
@@ -83,16 +84,21 @@ async def calculate_batch_cost_and_usage(
             pricing. Threaded through to batch_cost_calculator so that
             deployment-specific pricing (e.g. input_cost_per_token_batches)
             is used instead of the global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     first_row: Final = file_content_dictionary[0] if file_content_dictionary else None
     if _uses_native_vertex_output(custom_llm_provider, model_name, first_row):
-        return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info)
+        return calculate_vertex_ai_batch_cost_and_usage(
+            file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location
+        )
 
     return _aggregate_batch_cost_usage_models(
         entries=file_content_dictionary,
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -102,6 +108,7 @@ async def handle_completed_batch(
     model_name: str | None = None,
     litellm_params: dict[str, object] | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
@@ -115,6 +122,8 @@ async def handle_completed_batch(
         model_info: Optional deployment-level model info with custom pricing,
             threaded through so a deployment's configured rates win over the
             global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     # A completed batch whose request lines all failed has no output file - the
     # results are written to a separate error_file_id and output_file_id is None.
@@ -142,7 +151,10 @@ async def handle_completed_batch(
 
     output_file_result: Final = (
         calculate_vertex_ai_batch_cost_and_usage(
-            _iter_batch_output_entries(file_content), model_name, model_info=model_info
+            _iter_batch_output_entries(file_content),
+            model_name,
+            model_info=model_info,
+            vertex_location=vertex_location,
         )
         if _uses_native_vertex_output(
             custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None)
@@ -152,6 +164,7 @@ async def handle_completed_batch(
             custom_llm_provider=custom_llm_provider,
             model_name=model_name,
             model_info=model_info,
+            vertex_location=vertex_location,
         )
     )
 
@@ -190,6 +203,7 @@ def _classify_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
     """Classify every output line in a single pass, so counting failures never needs
     a second read of a potentially huge output file. A line the provider reported as
@@ -200,7 +214,7 @@ def _classify_output_line_stats(
         if not _batch_response_was_successful(entry, custom_llm_provider):
             yield _LineOutcome.PROVIDER_FAILED
             continue
-        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
         yield stats if stats is not None else _LineOutcome.UNCOSTABLE
 
 
@@ -209,13 +223,14 @@ def _safe_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats | None:
     """Return the stats for one provider-successful batch output line, or None when
     it cannot be costed, so a single bad line never aborts the whole batch's cost
     accounting."""
     custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
     try:
-        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
     except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
         verbose_logger.warning(
             "batch output line could not be costed, so it is billed at $0 and the rest of the batch "
@@ -231,6 +246,7 @@ def _compute_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats:
     response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
     usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
@@ -245,6 +261,7 @@ def _compute_output_line_stats(
         model_name=model_name,
         response_model=response_model,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
     return _BatchOutputLineStats(
         prompt_cost=line_prompt_cost,
@@ -274,6 +291,7 @@ def _output_line_cost(
     model_name: str | None,
     response_model: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """(prompt_cost, completion_cost) for one output line, priced at batch rates."""
     from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
@@ -294,6 +312,7 @@ def _output_line_cost(
         model=cost_model,
         custom_llm_provider=custom_llm_provider,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -302,11 +321,14 @@ def _aggregate_batch_cost_usage_models(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Aggregate cost, usage, models, and pass/fail counts from batch output
     entries in a single pass, holding one small stats record per line instead
     of the parsed file."""
-    all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+    all_results: Final = tuple(
+        _classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location)
+    )
     line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
     failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
     successful_requests: Final = len(all_results) - failed_requests
@@ -353,6 +375,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
     vertex_ai_batch_responses: Iterable[dict[str, object]],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Cost and usage of a native Vertex predictions.jsonl, one
@@ -371,6 +394,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
             model_info=model_info,
             calculate_usage=VertexGeminiConfig.calculate_usage,
             cost_calculator=batch_cost_calculator,
+            vertex_location=vertex_location,
         )
         for row in vertex_ai_batch_responses
     )
@@ -777,11 +801,17 @@ def _batch_response_was_successful(
     message batch results lines report ``result.type == "succeeded"``; Bedrock
     batch output lines report ``modelOutput`` (and no ``error``).
     """
-    if custom_llm_provider == "anthropic":
-        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
-        dict[str, object], batch_job_output_file.get("response", None) or {}
+    envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
+    envelope: Final = batch_job_output_file.get(envelope_key)
+    if not isinstance(envelope, Mapping):
+        if envelope is not None:
+            verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
+        return False
+    response: Final = cast(Mapping[object, object], envelope)  # cast-ok: mapping shape validated above
+    return (
+        response.get("type") == "succeeded"
+        if custom_llm_provider == "anthropic"
+        else response.get("status_code") == 200
     )
-    return _response.get("status_code", None) == 200

litellm/caching/redis_cache.py

@@ -400,12 +400,19 @@ def _redis_timeout_error_types() -> tuple[type, ...]:
     ``builtins.TimeoutError`` covers ``socket.timeout`` (an alias since py3.10) and, from
     py3.11, ``asyncio.TimeoutError``; on py3.10 ``asyncio.TimeoutError`` is still its own
     class, so it is listed explicitly. ``redis.exceptions.TimeoutError`` subclasses neither.
+    ``MaxConnectionsError`` is the async cluster client's pool-exhausted signal (its nodes have
+    no blocking pool, so it is raised instantly instead of after a wait): a busy pool, not a
+    dead Redis, like the standalone pool's chained wait timeout.
     """
     try:
-        from redis.exceptions import TimeoutError as RedisTimeoutError
+        from redis import exceptions as redis_exceptions
     except ImportError:
         return (TimeoutError, asyncio.TimeoutError)
-    return (RedisTimeoutError, TimeoutError, asyncio.TimeoutError)
+    max_connections_error: Final = cast(  # cast-ok: types-redis 4.6 stubs predate MaxConnectionsError
+        "type[Exception]",
+        redis_exceptions.MaxConnectionsError,  # pyright: ignore[reportAttributeAccessIssue]  # types-redis 4.6 stubs predate it
+    )
+    return (redis_exceptions.TimeoutError, TimeoutError, asyncio.TimeoutError, max_connections_error)
 
 
 _MAX_EXCEPTION_CAUSE_DEPTH: Final = 20
@@ -637,9 +644,14 @@ def _redis_circuit_breaker_guard(method):
     """
 
     @functools.wraps(method)
-    async def wrapper(self, *args, **kwargs):
+    async def wrapper(self: "RedisCache", *args, **kwargs):
+        async def guarded() -> object:
+            return await method(self, *args, **kwargs)
+
         return await _run_under_circuit_breaker(
-            self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
+            self._circuit_breaker,  # pyright: ignore[reportPrivateUsage]  # the guard is RedisCache's own decorator
+            wrapper.__name__,
+            lambda: self._admit_async_call(guarded),  # pyright: ignore[reportPrivateUsage]  # same decorator
         )
 
     return wrapper
@@ -725,6 +737,9 @@ class RedisCache(BaseCache):
         else:
             super().__init__()  # defaults to 60s
 
+    def _admit_async_call(self, call: Callable[[], Awaitable[_RedisCallResult]]) -> Awaitable[_RedisCallResult]:
+        return call()
+
     def _setup_health_pings(self):
         """Setup async and sync health pings for Redis."""
         # ASYNC HEALTH PING
@@ -797,6 +812,8 @@ class RedisCache(BaseCache):
     def init_async_client(
         self,
     ) -> async_redis_client | async_redis_cluster_client:
+        from redis.asyncio import RedisCluster
+
         from litellm import in_memory_llm_clients_cache
 
         from .._redis import get_redis_async_client, get_redis_connection_pool
@@ -809,7 +826,11 @@ class RedisCache(BaseCache):
             # Create new connection pool and client for current event loop
             self.async_redis_conn_pool = get_redis_connection_pool(**self.redis_kwargs)
             redis_async_client = get_redis_async_client(connection_pool=self.async_redis_conn_pool, **self.redis_kwargs)
-            in_memory_llm_clients_cache.set_cache(key=cache_key, value=redis_async_client)
+            in_memory_llm_clients_cache.set_cache(
+                key=cache_key,
+                value=redis_async_client,
+                litellm_owned_client=isinstance(redis_async_client, RedisCluster),
+            )
 
         self.redis_async_client = redis_async_client
         return redis_async_client
@@ -1093,7 +1114,9 @@ class RedisCache(BaseCache):
                     litellm.in_memory_llm_clients_cache.set_cache(key=script_cache_key, value=executor)
                 return await executor(keys=keys, args=args, client=client)
 
-            return await _run_under_circuit_breaker(self._circuit_breaker, "run_script", execute)
+            return await _run_under_circuit_breaker(
+                self._circuit_breaker, "run_script", lambda: self._admit_async_call(execute)
+            )
 
         return run_script
 

litellm/caching/redis_cluster_cache.py

@@ -5,13 +5,21 @@ Key differences:
 - RedisClient NEEDs to be re-used across requests, adds 3000ms latency if it's re-created
 """
 
-from typing import TYPE_CHECKING, Any, Final
+import asyncio
+import contextvars
+import os
+import weakref
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar
 
+from litellm._logging import verbose_logger
 from litellm.caching.redis_cache import RedisCache
+from litellm.constants import REDIS_CONNECTION_POOL_TIMEOUT
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
-    from redis.asyncio import Redis, RedisCluster
+    from redis.asyncio import Redis
     from redis.asyncio.client import Pipeline
 
     pipeline = Pipeline
@@ -23,25 +31,100 @@ else:
     Span = Any
 
 
+_T: Final = TypeVar("_T")
+
+
+def _connection_limit(max_connections: object) -> int | None:
+    try:
+        limit: Final = int(max_connections)  # pyright: ignore[reportArgumentType]  # config value, str or int
+    except (TypeError, ValueError):
+        return None
+    return limit if limit > 0 else None
+
+
+_ClientSemaphores: TypeAlias = weakref.WeakValueDictionary[str, asyncio.Semaphore]
+
+
+@dataclass(slots=True)
+class _SemaphoreRegistry:
+    """Weak on both layers: a semaphore binds to its loop once a waiter queues on it, so a strong value would keep a
+    closed loop alive. In-flight ``ClusterConnectionGate.run`` calls hold the semaphore they acquired."""
+
+    _per_loop: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, _ClientSemaphores]" = field(
+        default_factory=weakref.WeakKeyDictionary, repr=False
+    )
+
+    def get(self, client_key: str, limit: int) -> asyncio.Semaphore:
+        per_loop: Final = self._per_loop.setdefault(asyncio.get_running_loop(), weakref.WeakValueDictionary())
+        existing: Final = per_loop.get(client_key)
+        if existing is not None:
+            return existing
+        created: Final = asyncio.Semaphore(limit)
+        per_loop[client_key] = created
+        return created
+
+
+_Admission: TypeAlias = tuple[str, "asyncio.Task[object] | None"]
+_SEMAPHORES: Final = _SemaphoreRegistry()
+_ADMITTED: Final[contextvars.ContextVar[frozenset[_Admission]]] = contextvars.ContextVar(
+    "redis_cluster_gate_admitted", default=frozenset()
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ClusterConnectionGate:
+    """The async ``RedisCluster`` has no blocking pool, so past ``max_connections`` a node raises
+    ``MaxConnectionsError`` instead of waiting and the circuit breaker opens on a healthy Redis. This gate makes
+    guarded commands wait for a slot the way the standalone ``BlockingConnectionPool`` does. A command that finds no
+    slot within ``wait_timeout`` runs anyway, so a genuinely saturated pool still reaches the client's own error
+    handling and the breaker."""
+
+    limit: int
+    wait_timeout: float = REDIS_CONNECTION_POOL_TIMEOUT
+
+    @classmethod
+    def from_max_connections(cls, max_connections: object) -> "ClusterConnectionGate | None":
+        limit: Final = _connection_limit(max_connections)
+        return None if limit is None else cls(limit=limit)
+
+    async def run(self, call: Callable[[], Awaitable[_T]], client_key: str) -> _T:
+        admission: Final[_Admission] = (client_key, asyncio.current_task())
+        if admission in _ADMITTED.get():
+            return await call()
+        semaphore: Final = _SEMAPHORES.get(client_key, self.limit)
+        try:
+            await asyncio.wait_for(semaphore.acquire(), timeout=self.wait_timeout)
+        except asyncio.TimeoutError:
+            verbose_logger.debug(
+                "Redis Cluster: no free connection slot within %ss (max_connections=%s), sending anyway",
+                self.wait_timeout,
+                self.limit,
+            )
+            return await call()
+        token: Final = _ADMITTED.set(_ADMITTED.get() | {admission})
+        try:
+            return await call()
+        finally:
+            _ADMITTED.reset(token)
+            semaphore.release()
+
+
 class RedisClusterCache(RedisCache):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.redis_async_redis_cluster_client: RedisCluster | None = None
-        self.redis_sync_redis_cluster_client: RedisCluster | None = None
+    def __init__(self, *args, max_connections: object = None, connection_gate: object = None, **kwargs):
+        pool_kwargs: Final = {} if max_connections is None else {"max_connections": max_connections}
+        super().__init__(*args, **pool_kwargs, **kwargs)
+        self._connection_gate: Final = (
+            connection_gate
+            if isinstance(connection_gate, ClusterConnectionGate)
+            else ClusterConnectionGate.from_max_connections(
+                os.environ.get("REDIS_MAX_CONNECTIONS") if max_connections is None else max_connections
+            )
+        )
 
-    def init_async_client(self):
-        from redis.asyncio import RedisCluster
-
-        from .._redis import get_redis_async_client
-
-        if self.redis_async_redis_cluster_client:
-            return self.redis_async_redis_cluster_client
-
-        _redis_client: Final = get_redis_async_client(connection_pool=self.async_redis_conn_pool, **self.redis_kwargs)
-        if isinstance(_redis_client, RedisCluster):
-            self.redis_async_redis_cluster_client = _redis_client
-
-        return _redis_client
+    def _admit_async_call(self, call: Callable[[], Awaitable[_T]]) -> Awaitable[_T]:
+        if self._connection_gate is None:
+            return call()
+        return self._connection_gate.run(call, self._get_async_client_cache_key())
 
     def _run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
         """

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -74,7 +74,7 @@ if TYPE_CHECKING:
         ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
-    from litellm.types.utils import Choices
+    from litellm.types.utils import Choices, Message
 
 
 _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@@ -308,6 +308,67 @@ def _as_chat_reasoning_items(
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
+    shifted: Final = {
+        key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
+        for key, value in annotation.items()
+    }
+    return cast(ChatCompletionAnnotation, shifted)
+
+
+def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
+    annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
+    return annotations or ()
+
+
+def _message_reasoning_content(message: "Message") -> str | None:
+    reasoning_content: str | None = getattr(message, "reasoning_content", None)
+    return reasoning_content
+
+
+def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
+    reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
+    return reasoning_items or ()
+
+
+def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
+    for message, offset in zip(messages, offsets):
+        for annotation in _message_annotations(message):
+            yield _shift_annotation(annotation, offset)
+
+
+def _fold_choices_into_one(
+    choices: Sequence["Choices"],
+    tool_calls: Sequence[Mapping[str, object]],
+    trailing_reasoning_content: str | None,
+    trailing_reasoning_item: _BuiltReasoningItem | None,
+) -> "Choices":
+    """Clients that only read choices[0] must still see every text and tool call."""
+    from litellm.types.utils import Choices, Message
+
+    messages: Final = tuple(choice.message for choice in choices)
+    texts: Final = tuple(str(message.content or "") for message in messages)
+    offsets: Final = (0, *accumulate(len(text) for text in texts))
+    annotations: Final = tuple(_shifted_annotations(messages, offsets))
+    reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
+    reasoning_items: Final = (
+        *chain.from_iterable(_message_reasoning_items(m) for m in messages),
+        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+    )
+    return Choices(
+        message=Message(
+            role=messages[0].role if messages else "assistant",
+            content="".join(texts) if messages else None,
+            annotations=list(annotations) or None,
+            tool_calls=list(tool_calls) or None,
+            reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        ),
+        finish_reason="tool_calls" if tool_calls else "stop",
+        index=0,
+    )
+
+
 _ToolChoiceT = TypeVar("_ToolChoiceT")
 
 
@@ -951,47 +1012,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else:
                 pass  # don't fail request if item in list is not supported
 
-        if accumulated_tool_calls and choices:
-            last_choice: Final = choices[-1]
-            last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
-            last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
-            merged_reasoning_content: Final = (
-                " ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
-            )
-            merged_reasoning_items: Final = _as_chat_reasoning_items(
-                (
-                    *(last_reasoning_items or ()),
-                    *(() if pending_reasoning_item is None else (pending_reasoning_item,)),
-                )
-            )
-            merged_message: Final = Message(
-                role=last_choice.message.role,
-                content=last_choice.message.content,
-                annotations=getattr(last_choice.message, "annotations", None),
+        if not choices and not accumulated_tool_calls:
+            return choices
+        return [
+            _fold_choices_into_one(
+                choices=choices,
                 tool_calls=accumulated_tool_calls,
-                reasoning_content=merged_reasoning_content,
-                reasoning_items=merged_reasoning_items,
+                trailing_reasoning_content=reasoning_content,
+                trailing_reasoning_item=pending_reasoning_item,
             )
-            return [
-                *choices[:-1],
-                Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
-            ]
-
-        if accumulated_tool_calls:
-            msg = Message(
-                content=None,
-                tool_calls=accumulated_tool_calls,
-                reasoning_content=reasoning_content,
-                reasoning_items=cast(
-                    list[ChatCompletionReasoningItem] | None,
-                    ([pending_reasoning_item] if pending_reasoning_item is not None else None),
-                ),
-            )
-            choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
-            reasoning_content = None
-            pending_reasoning_item = None
-
-        return choices
+        ]
 
     @staticmethod
     def _build_empty_incomplete_choice(

litellm/constants.py

@@ -285,6 +285,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
 
 # Provider-specific API base URLs
 XAI_API_BASE: Final = "https://api.x.ai/v1"
+PLACEHOLDER_API_KEY: Final = "fake-api-key"
 OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE"
 OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY"
 OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0"
@@ -1587,6 +1588,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
 CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
+VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
@@ -1654,6 +1656,7 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = {
 # e.g., 'x-pass-anthropic-beta: value' becomes 'anthropic-beta: value'
 # Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.)
 PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-"
+TYPESAFE_DANGER_ZONE_HEADER_PREFIX: Final = "typesafe-danger-zone-"
 INTERNAL_KWARG_PREFIX: Final = "_litellm_"
 CONTROL_OPTIONS_KEY: Final = f"{INTERNAL_KWARG_PREFIX}control"
 
@@ -1925,6 +1928,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "public_model_groups",
     "public_model_groups_links",
     "cost_discount_config",
+    "apply_search_free_tier",
     "cost_margin_config",
     "block_requests_for_models_without_pricing",
     "budget_exceeded_throttle_percentage",

litellm/cost_calculator.py

@@ -8,7 +8,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
 from httpx import Response
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict, assert_never
 
 import litellm
@@ -38,6 +38,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     get_regional_uplift_multiplier,
     get_service_tier_cost_key,
     get_token_type_cost_breakdown,
+    get_vertex_regional_endpoint_uplift,
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
@@ -127,6 +128,7 @@ from litellm.types.utils import (
     TranscriptionUsageTokensObject,
     Usage,
     VectorStoreSearchResponse,
+    is_server_derived_pricing_key,
 )
 from litellm.utils import (
     CallTypes,
@@ -139,6 +141,8 @@ from litellm.utils import (
     TextCompletionResponse,
     TranscriptionResponse,
     cached_get_model_info_helper,
+    model_info_with_pricing_overrides,
+    runtime_model_cost_registrations,
     token_counter,
 )
 
@@ -417,6 +421,7 @@ def cost_per_token(
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
     custom_model_info: OCRPricing | None = None,
+    bedrock_model_info: ModelInfo | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -642,6 +647,7 @@ def cost_per_token(
             model=model,
             custom_llm_provider=custom_llm_provider,
             data_residency=data_residency,
+            vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
         if _transcription_usage_has_token_details(usage_block):
@@ -704,6 +710,15 @@ def cost_per_token(
     elif custom_llm_provider == "anthropic":
         return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "bedrock":
+        resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False)
+        if resolved_bedrock_info is not None:
+            return generic_cost_per_token(
+                model=model,
+                usage=usage_block,
+                custom_llm_provider="bedrock",
+                service_tier=service_tier,
+                model_info=resolved_bedrock_info,
+            )
         return bedrock_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "openai":
         return openai_cost_per_token(
@@ -819,6 +834,60 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
     )
 
 
+def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool:
+    return _cost_map_entry_prices_anything(entry) or any(
+        value is not None and field.startswith("cache_") and is_server_derived_pricing_key(field)
+        for field, value in entry.items()
+    )
+
+
+def _bedrock_pricing_info(
+    model: str | None,
+    invoked_model_id: str | None,
+    custom_llm_provider: str | None,
+    explicit_pricing: bool,
+) -> ModelInfo | None:
+    if explicit_pricing or custom_llm_provider != "bedrock" or not model:
+        return None
+    registration: Final = next(
+        (
+            (key, entry)
+            for key, entry in runtime_model_cost_registrations(model, "bedrock")
+            if _bedrock_registration_has_prices(entry)
+        ),
+        None,
+    )
+    profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
+    if profile is None or not _declares_rates(profile):
+        return _cost_map_model_info(registration[0], None) if registration is not None else None
+    if registration is None or registration[0] == profile["key"]:
+        return profile
+    return model_info_with_pricing_overrides(profile, registration[1])
+
+
+def _declares_rates(model_info: ModelInfo | None) -> bool:
+    entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
+    return entry is not None and _cost_map_entry_prices_anything(entry)
+
+
+_LOGGED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
+
+
+def _invoked_model_id_for_cost(
+    optional_params: object,
+    logging_obj: LitellmLoggingObject | None,
+) -> str | None:
+    from litellm.llms.bedrock.common_utils import extract_model_name_from_bedrock_arn
+
+    recorded: Final[object] = getattr(logging_obj, "model_call_details", None)
+    invoked: Final = _get_hidden_str_for_cost_calc(optional_params, "model_id") or _get_hidden_str_for_cost_calc(
+        recorded, "model_id"
+    )
+    return (
+        extract_model_name_from_bedrock_arn(invoked) if invoked is not None and invoked.startswith("arn:") else invoked
+    )
+
+
 def select_model_name_for_cost_calc(
     model: str | None,
     completion_response: object | None,
@@ -827,6 +896,7 @@ def select_model_name_for_cost_calc(
     custom_llm_provider: str | None = None,
     router_model_id: str | None = None,
     region_name: str | None = None,
+    invoked_model_id: str | None = None,
 ) -> str | None:
     """
     1. If custom pricing is true, return received model name
@@ -847,6 +917,13 @@ def select_model_name_for_cost_calc(
     hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
+    configured_model: Final = model or (
+        completion_response_model if isinstance(completion_response_model, str) else None
+    )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        configured_model, invoked_model_id, custom_llm_provider, explicit_pricing
+    )
+    bedrock_pricing_model: Final = bedrock_model_info["key"] if bedrock_model_info is not None else None
     priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
     priced_region: Final = (
         _get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name
@@ -864,8 +941,8 @@ def select_model_name_for_cost_calc(
         else:
             return_model = model
 
-    elif base_model is not None or provider_response_model is not None:
-        return_model = base_model if base_model is not None else provider_response_model
+    elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
+        return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
 
     elif completion_response_model is None and hidden_params is not None:
         if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
@@ -1472,6 +1549,10 @@ def completion_cost(
             )
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
+        invoked_model_id: Final = _invoked_model_id_for_cost(
+            _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None,
+            litellm_logging_obj,
+        )
         selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
@@ -1480,8 +1561,15 @@ def completion_cost(
             base_model=base_model,
             router_model_id=router_model_id,
             region_name=region_name,
+            invoked_model_id=invoked_model_id,
         )
 
+        bedrock_model_info: Final = _bedrock_pricing_info(
+            model or _get_response_model(completion_response),
+            invoked_model_id,
+            _get_provider_for_cost_calc(model, custom_llm_provider),
+            explicit_pricing,
+        )
         potential_model_names: Final = [
             selected_model,
             _get_response_model(completion_response),
@@ -1844,6 +1932,7 @@ def completion_cost(
                     vertex_location=vertex_location,
                     response=completion_response,
                     custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
+                    bedrock_model_info=bedrock_model_info,
                 )
 
                 # Get additional costs from provider (e.g., routing fees, infrastructure costs)
@@ -1932,6 +2021,7 @@ def completion_cost(
                             data_residency=data_residency,
                             vertex_location=vertex_location,
                             custom_cost_per_token=custom_cost_per_token,
+                            model_info=bedrock_model_info,
                         )
                         _reasoning_cost = _token_type_breakdown.reasoning_cost
                         _cache_read_cost = _token_type_breakdown.cache_read_cost
@@ -2026,6 +2116,8 @@ def response_cost_calculator(
         "asearch",
         "decisions",
         "adecisions",
+        "systemone",
+        "asystemone",
     ],
     optional_params: dict,
     cache_hit: bool | None = None,
@@ -2174,7 +2266,16 @@ def pricing_entry_for_cost_calc(
         custom_llm_provider=custom_llm_provider,
         router_model_id=router_model_id,
         region_name=region_name,
+        invoked_model_id=_invoked_model_id_for_cost(None, litellm_logging_obj),
     )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        model or _get_response_model(completion_response),
+        _invoked_model_id_for_cost(None, litellm_logging_obj),
+        _get_provider_for_cost_calc(model, custom_llm_provider),
+        custom_pricing is True or base_model is not None,
+    )
+    if bedrock_model_info is not None:
+        return bedrock_model_info["key"], bedrock_model_info
     candidates: Final = (selected_model, _get_response_model(completion_response), model)
     resolved: Final = next(
         (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info),
@@ -2622,6 +2723,8 @@ def batch_cost_calculator(
     custom_llm_provider: str | None = None,
     model_info: ModelInfo | None = None,
     data_residency: str | None = None,
+    *,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """
     Calculate the cost of a batch job.
@@ -2711,7 +2814,8 @@ def batch_cost_calculator(
         total_prompt_cost *= uplift
         total_completion_cost *= uplift
 
-    return total_prompt_cost, total_completion_cost
+    vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
+    return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
 
 
 def _batch_prompt_cost(

litellm/decisions/call.py

@@ -0,0 +1,795 @@
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from functools import partial
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
+
+import httpx
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
+from typing_extensions import assert_never
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    async_convert_urls_to_base64,
+    convert_url_to_base64,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.decisions.transformation import (
+    BaseDecisionsConfig,
+    image_media_type,
+    image_param,
+    input_images,
+    ir_to_systemone_response,
+    remote_image_urls,
+    systemone_request_to_ir,
+    with_images_where,
+    with_inlined_images,
+)
+from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
+from litellm.types.decisions import (
+    DecisionQuestion,
+    DecisionsAsset,
+    DecisionsAssetReference,
+    DecisionsAssets,
+    DecisionsAudio,
+    DecisionsBase64Image,
+    DecisionsImage,
+    DecisionsIRAnswer,
+    DecisionsIRBoxQuestion,
+    DecisionsIRMessages,
+    DecisionsIRQuestion,
+    DecisionsIRRefusal,
+    DecisionsIRRequest,
+    DecisionsIRResponse,
+    DecisionsIRState,
+    DecisionsJSON,
+    DecisionsRequestBody,
+    DecisionsResponse,
+    DecisionsVideo,
+    OpenAIDecisionInput,
+    OpenAIDecisionQuestion,
+    OpenAIDecisionRequestBody,
+    OpenAIDecisionResponse,
+    UnsupportedDecisionsRequest,
+)
+from litellm.types.llms.openai import ChatCompletionUserMessage
+from litellm.types.utils import LlmProviders, ProviderSpecificHeader
+from litellm.utils import ProviderConfigManager
+
+SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
+OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
+DecisionsImages: TypeAlias = Sequence[str | DecisionsBase64Image | Mapping[str, object]]
+DecisionsAssetEntries: TypeAlias = Mapping[str, DecisionsAsset | Mapping[str, object]]
+DecisionsAudioClips: TypeAlias = Sequence[DecisionsAudio | Mapping[str, object]]
+DecisionsVideoClips: TypeAlias = Sequence[DecisionsVideo | Mapping[str, object]]
+DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
+
+_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
+_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody)
+_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(
+    str | None, config=ConfigDict(title="safety_identifier")
+)
+_IMAGES_ADAPTER: Final[TypeAdapter[tuple[DecisionsImage, ...] | None]] = TypeAdapter(
+    tuple[DecisionsImage, ...] | None, config=ConfigDict(title="images")
+)
+_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[
+    TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None]
+] = TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None)
+_HANDLER: Final = BaseLLMHTTPHandler()
+_BASE64_MARKER: Final = ";base64,"
+
+
+_IMAGE_ADAPTER: Final[TypeAdapter[DecisionsImage]] = TypeAdapter(DecisionsImage, config=ConfigDict(title="images"))
+_IMAGE_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
+_ASSETS_ADAPTER: Final[TypeAdapter[DecisionsAssets | None]] = TypeAdapter(
+    DecisionsAssets | None, config=ConfigDict(title="assets")
+)
+_ASSET_ENTRY_ADAPTER: Final[TypeAdapter[tuple[DecisionsAssetReference, DecisionsAsset]]] = TypeAdapter(
+    tuple[DecisionsAssetReference, DecisionsAsset]
+)
+_ASSET_ENTRIES_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _valid_image(image: object) -> DecisionsImage | None:
+    try:
+        return _IMAGE_ADAPTER.validate_python(image)
+    except ValidationError:
+        return None
+
+
+def _image_entries(images: object) -> tuple[object, ...]:
+    try:
+        return _IMAGE_ENTRIES_ADAPTER.validate_python(images)
+    except ValidationError:
+        return ()
+
+
+def _is_well_formed_image(image: DecisionsImage) -> bool:
+    return _valid_image(image) is not None
+
+
+def _no_image(_image: DecisionsImage) -> bool:
+    return False
+
+
+def _valid_asset(entry: tuple[str, object]) -> tuple[str, DecisionsAsset] | None:
+    try:
+        return _ASSET_ENTRY_ADAPTER.validate_python(entry)
+    except ValidationError:
+        return None
+
+
+def _asset_entries(assets: object) -> Mapping[str, object]:
+    try:
+        return _ASSET_ENTRIES_ADAPTER.validate_python(assets)
+    except ValidationError:
+        return MappingProxyType({})
+
+
+def _request_assets(assets: object, drops_params: bool) -> Mapping[str, DecisionsAsset] | None:
+    if not drops_params:
+        return _ASSETS_ADAPTER.validate_python(assets)
+    valid: Final = MappingProxyType(
+        dict(entry for entry in map(_valid_asset, _asset_entries(assets).items()) if entry is not None)
+    )
+    return valid or None
+
+
+def _request_images(images: object, drops_params: bool) -> tuple[DecisionsImage, ...]:
+    if not drops_params:
+        return _IMAGES_ADAPTER.validate_python(images) or ()
+    return tuple(image for image in map(_valid_image, _image_entries(images)) if image is not None)
+
+
+@dataclass(frozen=True, slots=True)
+class SystemOneRequestFields:
+    state: DecisionsJSON | None
+    questions: SystemOneQuestions | None
+    images: object = None
+    assets: object = None
+    audio: DecisionsAudioClips | None = None
+    videos: DecisionsVideoClips | None = None
+
+    def validate(self, *, safety_identifier: str | None, drops_params: bool) -> DecisionsRequestBody:
+        return _SYSTEMONE_REQUEST_ADAPTER.validate_python(
+            {
+                "state": self.state,
+                "questions": self.questions,
+                "images": _request_images(self.images, drops_params) or None,
+                "assets": _request_assets(self.assets, drops_params),
+                "audio": self.audio,
+                "videos": self.videos,
+            }
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class OpenAIDecisionRequestFields:
+    input: OpenAIDecisionInput | None
+    questions: OpenAIDecisionQuestions | None
+    audio: DecisionsAudioClips | None = None
+    videos: DecisionsVideoClips | None = None
+
+    def validate(self, *, safety_identifier: str | None, drops_params: bool) -> OpenAIDecisionRequestBody:
+        return _OPENAI_REQUEST_ADAPTER.validate_python(
+            {
+                "input": self.input,
+                "questions": self.questions,
+                "safety_identifier": safety_identifier,
+                "audio": self.audio,
+                "videos": self.videos,
+            }
+        )
+
+
+RequestFields: TypeAlias = SystemOneRequestFields | OpenAIDecisionRequestFields
+
+
+@dataclass(frozen=True, slots=True, repr=False)
+class DecisionsCall:
+    model: str
+    requested_model: str
+    custom_llm_provider: str
+    provider_config: BaseDecisionsConfig
+    request: DecisionsRequestFormat
+    ir_request: DecisionsIRRequest
+    api_base: str
+    api_key: str | None = field(repr=False)
+    logging_obj: LiteLLMLoggingObj | None
+    headers: Mapping[str, str]
+    timeout: float | httpx.Timeout | None
+    litellm_params: Mapping[str, object] = field(repr=False)
+    caller_questions: tuple[DecisionsIRQuestion, ...]
+
+
+def supported_decisions_providers() -> tuple[str, ...]:
+    return tuple(
+        provider.value
+        for provider in LlmProviders
+        if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
+    )
+
+
+def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
+    provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
+    provider_config: Final = (
+        None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
+    )
+    if provider_config is None:
+        supported: Final = ", ".join(supported_decisions_providers())
+        raise litellm.BadRequestError(
+            message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
+            model=model,
+            llm_provider=custom_llm_provider,
+        )
+    return provider_config
+
+
+def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest:
+    match request:
+        case DecisionsRequestBody():
+            return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier)
+        case OpenAIDecisionRequestBody():
+            return openai_request_to_ir(request)
+        case _:
+            assert_never(request)
+
+
+def _drops_params(kwargs: Mapping[str, object]) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
+
+
+def _request_safety_identifier(
+    safety_identifier: object, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object]
+) -> str | None:
+    if not _drops_params(kwargs):
+        return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier)
+    if isinstance(safety_identifier, str) and provider_config.supports_safety_identifier:
+        return safety_identifier
+    return None
+
+
+def _fields_without_unsupported_media(
+    fields: RequestFields,
+    provider_config: BaseDecisionsConfig,
+    model: str,
+    custom_llm_provider: str,
+    kwargs: Mapping[str, object],
+) -> RequestFields:
+    if not _drops_params(kwargs):
+        return fields
+    return replace(
+        fields,
+        audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
+        videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _UnsupportedParams:
+    names: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _NothingLeftAfterDrop:
+    names: tuple[str, ...]
+    missing: Literal["input to decide on", "questions to answer"]
+
+
+@dataclass(frozen=True, slots=True)
+class _MalformedImages:
+    count: int
+
+
+def _unsupported_params(
+    ir_request: DecisionsIRRequest,
+    provider_config: BaseDecisionsConfig,
+    model: str,
+    custom_llm_provider: str,
+    headers: Mapping[str, str],
+) -> tuple[str, ...]:
+    refuses_safety_identifier: Final = (
+        ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
+    )
+    refused_image_param: Final = (
+        None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
+    )
+    refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
+    refuses_box_questions: Final = _has_box_question(ir_request.questions) and not (
+        provider_config.supports_box_questions(model, custom_llm_provider, headers)
+    )
+    refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
+        model, custom_llm_provider
+    )
+    refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
+        model, custom_llm_provider
+    )
+    return (
+        *(("safety_identifier",) if refuses_safety_identifier else ()),
+        *(() if refused_image_param is None else (refused_image_param,)),
+        *(("assets",) if refuses_assets else ()),
+        *(("box questions",) if refuses_box_questions else ()),
+        *(("audio",) if refuses_audio else ()),
+        *(("videos",) if refuses_videos else ()),
+    )
+
+
+def _has_box_question(questions: Sequence[DecisionsIRQuestion]) -> bool:
+    return any(isinstance(question, DecisionsIRBoxQuestion) for question in questions)
+
+
+def _is_not_box_question(question: DecisionsIRQuestion) -> bool:
+    return not isinstance(question, DecisionsIRBoxQuestion)
+
+
+def _without_unsupported_params(
+    ir_request: DecisionsIRRequest,
+    provider_config: BaseDecisionsConfig,
+    model: str,
+    custom_llm_provider: str,
+    headers: Mapping[str, str],
+) -> DecisionsIRRequest:
+    return replace(
+        ir_request,
+        input=(
+            ir_request.input
+            if provider_config.supports_images_in(ir_request.input, headers)
+            else with_images_where(ir_request.input, _no_image)
+        ),
+        safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
+        assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
+        questions=(
+            ir_request.questions
+            if provider_config.supports_box_questions(model, custom_llm_provider, headers)
+            else tuple(filter(_is_not_box_question, ir_request.questions))
+        ),
+    )
+
+
+def _leaves_no_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> bool:
+    return isinstance(decision_input, DecisionsIRMessages) and not decision_input.messages
+
+
+def _without_malformed_images(
+    ir_request: DecisionsIRRequest, drops_params: bool
+) -> DecisionsIRRequest | _MalformedImages:
+    malformed: Final = sum(1 for image in input_images(ir_request.input) if not _is_well_formed_image(image))
+    if not malformed:
+        return ir_request
+    well_formed: Final = replace(ir_request, input=with_images_where(ir_request.input, _is_well_formed_image))
+    if not drops_params or _leaves_no_input(well_formed.input):
+        return _MalformedImages(count=malformed)
+    return well_formed
+
+
+def _provider_ir_request(
+    ir_request: DecisionsIRRequest,
+    *,
+    provider_config: BaseDecisionsConfig,
+    model: str,
+    custom_llm_provider: str,
+    headers: Mapping[str, str],
+    kwargs: Mapping[str, object],
+) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
+    unsupported: Final = _unsupported_params(ir_request, provider_config, model, custom_llm_provider, headers)
+    if not unsupported:
+        return ir_request
+    if not _drops_params(kwargs):
+        return _UnsupportedParams(names=unsupported)
+    supported: Final = _without_unsupported_params(ir_request, provider_config, model, custom_llm_provider, headers)
+    if _leaves_no_input(supported.input):
+        return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
+    if not supported.questions:
+        return _NothingLeftAfterDrop(names=unsupported, missing="questions to answer")
+    return supported
+
+
+def _provider_specific_header(
+    value: object,
+) -> ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None:
+    try:
+        return _PROVIDER_SPECIFIC_HEADER_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _outbound_headers(
+    extra_headers: Mapping[str, str] | None, custom_llm_provider: str, kwargs: Mapping[str, object]
+) -> Mapping[str, str]:
+    forwarded: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
+        provider_specific_header=_provider_specific_header(kwargs.get("provider_specific_header")),
+        custom_llm_provider=custom_llm_provider,
+    )
+    return MappingProxyType({**(extra_headers or {}), **forwarded})
+
+
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64")
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
+def prepare_call(
+    *,
+    model: str,
+    fields: RequestFields,
+    safety_identifier: str | None,
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None,
+    custom_llm_provider: str | None,
+    extra_headers: Mapping[str, str] | None,
+    kwargs: Mapping[str, object],
+) -> DecisionsCall:
+    upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        api_base=api_base,
+        api_key=api_key,
+    )
+    provider_config: Final = _provider_config(upstream_model, provider)
+    if not upstream_model:
+        raise litellm.BadRequestError(
+            message="A model name is required for the Decisions API",
+            model=model,
+            llm_provider=provider,
+        )
+    try:
+        canonical_model: Final = provider_config.canonical_model(upstream_model)
+    except ValueError as error:
+        raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
+    try:
+        headers: Final = _outbound_headers(extra_headers, provider, kwargs)
+        request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
+        request: Final = _fields_without_unsupported_media(
+            fields, provider_config, canonical_model, provider, kwargs
+        ).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
+    except ValidationError as error:
+        raise litellm.BadRequestError(
+            message=f"Invalid Decisions request: {error}",
+            model=model,
+            llm_provider=provider,
+        ) from error
+
+    resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
+    if resolved_api_base is None:
+        raise litellm.BadRequestError(
+            message=provider_config.missing_api_base_message(provider),
+            model=model,
+            llm_provider=provider,
+        )
+    resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
+    if resolved_api_key is None and provider_config.api_key_required:
+        raise litellm.AuthenticationError(
+            message=f"Missing API key for Decisions provider '{provider}'",
+            model=model,
+            llm_provider=provider,
+        )
+
+    caller_ir_request: Final = _without_malformed_images(
+        _ir_request(request, request_safety_identifier), _drops_params(kwargs)
+    )
+    if isinstance(caller_ir_request, _MalformedImages):
+        raise litellm.BadRequestError(
+            message=(
+                f"Invalid Decisions request: {caller_ir_request.count} input_image part(s) have an image_url that is "
+                "neither a data:image/...;base64, URL nor an http(s) URL"
+            ),
+            model=model,
+            llm_provider=provider,
+        )
+    ir_request: Final = _provider_ir_request(
+        caller_ir_request,
+        provider_config=provider_config,
+        model=canonical_model,
+        custom_llm_provider=provider,
+        headers=headers,
+        kwargs=kwargs,
+    )
+    if isinstance(ir_request, _UnsupportedParams):
+        raise litellm.UnsupportedParamsError(
+            message=(
+                f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}. "
+                "To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
+                "`litellm_settings:\n drop_params: true`\n"
+            ),
+            model=model,
+            llm_provider=provider,
+        )
+    if isinstance(ir_request, _NothingLeftAfterDrop):
+        raise litellm.BadRequestError(
+            message=(
+                f"{provider} does not support parameters: {list(ir_request.names)}, for model={model}, "
+                f"and dropping them leaves no {ir_request.missing}"
+            ),
+            model=model,
+            llm_provider=provider,
+        )
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_from_kwargs(
+            kwargs=dict(kwargs),
+            model=canonical_model,
+            litellm_params={
+                "litellm_call_id": kwargs.get("litellm_call_id"),
+                "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+            },
+            custom_llm_provider=provider,
+        )
+    return DecisionsCall(
+        model=canonical_model,
+        requested_model=model,
+        custom_llm_provider=provider,
+        provider_config=provider_config,
+        request=request,
+        ir_request=ir_request,
+        api_base=resolved_api_base,
+        api_key=resolved_api_key,
+        logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
+        headers=headers,
+        timeout=timeout,
+        litellm_params=kwargs,
+        caller_questions=caller_ir_request.questions,
+    )
+
+
+def _remote_image_urls(call: DecisionsCall) -> tuple[str, ...]:
+    if not call.provider_config.inlines_remote_images:
+        return ()
+    return remote_image_urls(call.ir_request.input)
+
+
+def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> DecisionsCall:
+    return replace(
+        call, ir_request=replace(call.ir_request, input=with_inlined_images(call.ir_request.input, data_urls))
+    )
+
+
+def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
+    remote_urls: Final = _remote_image_urls(call)
+    if not remote_urls:
+        return call
+    return _with_data_urls(
+        call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls})
+    )
+
+
+async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
+    remote_urls: Final = _remote_image_urls(call)
+    if not remote_urls:
+        return call
+    data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
+    return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
+
+
+def _has_media_type_in(media_types: frozenset[str], image: DecisionsImage) -> bool:
+    media_type: Final = image_media_type(image)
+    return media_type is None or media_type in media_types
+
+
+def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
+    media_types: Final = call.provider_config.image_media_types
+    if media_types is None:
+        return call
+    found: Final = (image_media_type(image) for image in input_images(call.ir_request.input))
+    unsupported: Final = tuple(
+        dict.fromkeys(type_ for type_ in found if type_ is not None and type_ not in media_types)
+    )
+    if not unsupported:
+        return call
+    supported: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
+    if _drops_params(call.litellm_params) and not _leaves_no_input(supported):
+        return replace(call, ir_request=replace(call.ir_request, input=supported))
+    raise litellm.BadRequestError(
+        message=(
+            f"{call.custom_llm_provider} takes only {', '.join(sorted(media_types))} images, "
+            f"got {', '.join(unsupported)}"
+        ),
+        model=call.requested_model,
+        llm_provider=call.custom_llm_provider,
+    )
+
+
+def _checked_image_count(call: DecisionsCall) -> DecisionsCall:
+    max_images: Final = call.provider_config.max_images
+    image_count: Final = len(input_images(call.ir_request.input))
+    if max_images is None or image_count <= max_images:
+        return call
+    raise litellm.BadRequestError(
+        message=f"{call.custom_llm_provider} takes at most {max_images} images per request, got {image_count}",
+        model=call.requested_model,
+        llm_provider=call.custom_llm_provider,
+    )
+
+
+def _decoded_image_bytes(image: DecisionsImage) -> int:
+    if isinstance(image, DecisionsBase64Image):
+        return len(image.base64.rstrip("=")) * 3 // 4
+    marker_at: Final = image.lower().find(_BASE64_MARKER)
+    return 0 if marker_at < 0 else len(image[marker_at + len(_BASE64_MARKER) :].rstrip("=")) * 3 // 4
+
+
+def _byte_limit(limit: int) -> str:
+    return f"{limit:,} bytes ({limit / (1024 * 1024):g} MiB)"
+
+
+def _image_size_error(call: DecisionsCall) -> str | None:
+    sizes: Final = tuple(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
+    largest: Final = max(sizes, default=0)
+    per_image: Final = call.provider_config.max_image_bytes
+    total: Final = call.provider_config.max_total_image_bytes
+    if per_image is not None and largest > per_image:
+        return f"takes images of at most {_byte_limit(per_image)} each, got one of {largest:,} bytes"
+    if total is not None and sum(sizes) > total:
+        return f"takes at most {_byte_limit(total)} of images per request, got {sum(sizes):,} bytes"
+    return None
+
+
+def _checked_image_sizes(call: DecisionsCall) -> DecisionsCall:
+    error: Final = _image_size_error(call)
+    if error is None:
+        return call
+    raise litellm.BadRequestError(
+        message=f"{call.custom_llm_provider} {error}",
+        model=call.requested_model,
+        llm_provider=call.custom_llm_provider,
+    )
+
+
+def _request_body(call: DecisionsCall) -> Mapping[str, object]:
+    body: Final = call.provider_config.transform_decisions_request(
+        model=call.model, request=call.ir_request, custom_llm_provider=call.custom_llm_provider
+    )
+    if isinstance(body, UnsupportedDecisionsRequest):
+        raise litellm.BadRequestError(
+            message=f"Decisions provider '{call.custom_llm_provider}' cannot serve this request: {body.reason}",
+            model=call.requested_model,
+            llm_provider=call.custom_llm_provider,
+        )
+    return body
+
+
+def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    formatted: Final = _formatted_response(response, call)
+    provider_cost: Final = call.provider_config.provider_reported_cost(response)
+    formatted.set_hidden_params(
+        {
+            "model": f"{call.custom_llm_provider}/{call.model}",
+            "custom_llm_provider": call.custom_llm_provider,
+            "provider_response_model": f"{call.custom_llm_provider}/{call.model}",
+            **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
+        }
+    )
+    return formatted
+
+
+def _caller_answers(call: DecisionsCall, answers: Sequence[DecisionsIRAnswer]) -> tuple[DecisionsIRAnswer, ...]:
+    sent_answers: Final = iter(answers)
+    return tuple(
+        next(sent_answers) if question in call.ir_request.questions else DecisionsIRRefusal()
+        for question in call.caller_questions
+    )
+
+
+def _formatted_response(
+    response: DecisionsIRResponse, call: DecisionsCall
+) -> DecisionsResponse | OpenAIDecisionResponse:
+    caller_request: Final = replace(call.ir_request, questions=call.caller_questions)
+    caller_response: Final = replace(response, answers=_caller_answers(call, response.answers))
+    match call.request:
+        case DecisionsRequestBody():
+            return ir_to_systemone_response(caller_response, caller_request)
+        case OpenAIDecisionRequestBody():
+            return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+        case _:
+            assert_never(call.request)
+
+
+def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
+    if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
+        provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
+        return litellm.APIConnectionError(
+            message=f"{provider_label} - {error.message}",
+            llm_provider=call.custom_llm_provider,
+            model=f"{call.custom_llm_provider}/{call.model}",
+        )
+    return litellm.exception_type(
+        model=f"{call.custom_llm_provider}/{call.model}",
+        custom_llm_provider=call.custom_llm_provider,
+        original_exception=error,
+    )
+
+
+async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    supported: Final = _checked_image_count(_checked_image_types(call))
+    inlined: Final = _checked_image_sizes(_checked_image_types(await _async_with_remote_images_inlined(supported)))
+    body: Final = _request_body(inlined)
+    try:
+        response: Final = await _HANDLER.adecisions(
+            model=inlined.model,
+            custom_llm_provider=inlined.custom_llm_provider,
+            logging_obj=inlined.logging_obj,
+            provider_config=inlined.provider_config,
+            request=inlined.ir_request,
+            body=body,
+            logged_body=body if inlined is supported else _request_body(supported),
+            api_base=inlined.api_base,
+            api_key=inlined.api_key,
+            headers=inlined.headers,
+            timeout=inlined.timeout,
+            litellm_params=inlined.litellm_params,
+        )
+    except Exception as error:
+        raise _map_upstream_exception(error, inlined) from error
+    return _format_response(response, inlined)
+
+
+def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    supported: Final = _checked_image_count(_checked_image_types(call))
+    inlined: Final = _checked_image_sizes(_checked_image_types(_with_remote_images_inlined(supported)))
+    body: Final = _request_body(inlined)
+    try:
+        response: Final = _HANDLER.decisions(
+            model=inlined.model,
+            custom_llm_provider=inlined.custom_llm_provider,
+            logging_obj=inlined.logging_obj,
+            provider_config=inlined.provider_config,
+            request=inlined.ir_request,
+            body=body,
+            logged_body=body if inlined is supported else _request_body(supported),
+            api_base=inlined.api_base,
+            api_key=inlined.api_key,
+            headers=inlined.headers,
+            timeout=inlined.timeout,
+            litellm_params=inlined.litellm_params,
+        )
+    except Exception as error:
+        raise _map_upstream_exception(error, inlined) from error
+    return _format_response(response, inlined)
+
+
+def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None:
+    if field_name not in kwargs:
+        return
+    raise litellm.BadRequestError(
+        message=f"Requests with '{field_name}' go through litellm.{other_function}",
+        model=model,
+        llm_provider="",
+    )
+
+
+def reject_images_next_to_input(*, model: str, kwargs: Mapping[str, object]) -> None:
+    if not kwargs.get("images") or _drops_params(kwargs):
+        return
+    raise litellm.BadRequestError(
+        message=(
+            "images belongs to the System One format; with input (OpenAI format), "
+            "send images as input_image content parts"
+        ),
+        model=model,
+        llm_provider="",
+    )

litellm/decisions/main.py

@@ -1,267 +1,127 @@
-from collections.abc import Mapping, Sequence
-from dataclasses import dataclass, field
-from typing import Final, TypeAlias
+import warnings
+from collections.abc import Mapping
+from typing import Final
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
-from typing_extensions import assert_never
+from pydantic import TypeAdapter
 
-import litellm
-from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.base_llm.chat.transformation import BaseLLMException
-from litellm.llms.base_llm.decisions.transformation import (
-    BaseDecisionsConfig,
-    ir_to_systemone_response,
-    systemone_request_to_ir,
+from litellm.decisions.call import (
+    DecisionsAudioClips,
+    DecisionsCall,
+    DecisionsVideoClips,
+    OpenAIDecisionQuestions,
+    OpenAIDecisionRequestFields,
+    RequestFields,
+    SystemOneQuestions,
+    SystemOneRequestFields,
+    asend,
+    log_caller_request,
+    prepare_call,
+    reject_images_next_to_input,
+    send,
 )
-from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
-from litellm.types.decisions import (
-    DecisionQuestion,
-    DecisionsIRRequest,
-    DecisionsIRResponse,
-    DecisionsJSON,
-    DecisionsRequestBody,
-    DecisionsResponse,
-    OpenAIDecisionInput,
-    OpenAIDecisionQuestion,
-    OpenAIDecisionRequestBody,
-    OpenAIDecisionResponse,
-    UnsupportedDecisionsRequest,
+from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse
+from litellm.utils import client
+
+_STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None)
+_SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None)
+_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter(
+    OpenAIDecisionQuestions | None
 )
-from litellm.types.utils import LlmProviders
-from litellm.utils import ProviderConfigManager, client
-
-DecisionsQuestions: TypeAlias = (
-    Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
-)
-DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
-
-_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
-_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody)
-_HANDLER: Final = BaseLLMHTTPHandler()
+_SYSTEMONE_FIELDS: Final = frozenset({"state", "images", "assets"})
 
 
-@dataclass(frozen=True, slots=True, repr=False)
-class _DecisionsCall:
-    model: str
-    requested_model: str
-    custom_llm_provider: str
-    provider_config: BaseDecisionsConfig
-    request: DecisionsRequestFormat
-    ir_request: DecisionsIRRequest
-    body: Mapping[str, object] = field(repr=False)
-    api_base: str
-    api_key: str | None = field(repr=False)
-    logging_obj: LiteLLMLoggingObj | None
-    headers: Mapping[str, str]
-    timeout: float | httpx.Timeout | None
-
-
-def _supported_providers() -> tuple[str, ...]:
-    return tuple(
-        provider.value
-        for provider in LlmProviders
-        if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
-    )
-
-
-def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
-    provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
-    provider_config: Final = (
-        None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
-    )
-    if provider_config is None:
-        supported: Final = ", ".join(_supported_providers())
-        raise litellm.BadRequestError(
-            message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
-            model=model,
-            llm_provider=custom_llm_provider,
-        )
-    return provider_config
-
-
-def _validate_request(
-    *,
-    state: DecisionsJSON | None,
-    questions: DecisionsQuestions | None,
+def _request_fields(
+    model: str,
     decision_input: OpenAIDecisionInput | None,
-    safety_identifier: str | None,
-) -> DecisionsRequestFormat:
-    if decision_input is None:
-        return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions})
-    return _OPENAI_REQUEST_ADAPTER.validate_python(
-        {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier}
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
+    audio: DecisionsAudioClips | None,
+    videos: DecisionsVideoClips | None,
+    kwargs: Mapping[str, object],
+) -> RequestFields:
+    if "state" not in kwargs:
+        reject_images_next_to_input(model=model, kwargs=kwargs)
+        return OpenAIDecisionRequestFields(
+            input=decision_input,
+            questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions),
+            audio=audio,
+            videos=videos,
+        )
+    warnings.warn(
+        "litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format",
+        UserWarning,
+        stacklevel=5,
+    )
+    return SystemOneRequestFields(
+        state=_STATE_ADAPTER.validate_python(kwargs["state"]),
+        questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions),
+        images=kwargs.get("images"),
+        assets=kwargs.get("assets"),
+        audio=audio,
+        videos=videos,
     )
 
 
-def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest:
-    match request:
-        case DecisionsRequestBody():
-            return systemone_request_to_ir(request)
-        case OpenAIDecisionRequestBody():
-            return openai_request_to_ir(request)
-        case _:
-            assert_never(request)
-
-
-def _prepare_call(
+def _prepare_decisions_call(
     *,
     model: str,
-    state: DecisionsJSON | None,
-    questions: DecisionsQuestions | None,
     decision_input: OpenAIDecisionInput | None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
     safety_identifier: str | None,
+    audio: DecisionsAudioClips | None,
+    videos: DecisionsVideoClips | None,
     api_key: str | None,
     api_base: str | None,
     timeout: float | httpx.Timeout | None,
     custom_llm_provider: str | None,
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
-) -> _DecisionsCall:
-    upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
-        model=model,
-        custom_llm_provider=custom_llm_provider,
-        api_base=api_base,
-        api_key=api_key,
-    )
-    provider_config: Final = _provider_config(upstream_model, provider)
-    canonical_model: Final = provider_config.canonical_model(upstream_model)
-    if not upstream_model:
-        raise litellm.BadRequestError(
-            message="A model name is required for the Decisions API",
-            model=model,
-            llm_provider=provider,
-        )
-    if state is not None and decision_input is not None:
-        raise litellm.BadRequestError(
-            message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both",
-            model=model,
-            llm_provider=provider,
-        )
-    try:
-        request: Final = _validate_request(
-            state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier
-        )
-    except ValidationError as error:
-        raise litellm.BadRequestError(
-            message=f"Invalid Decisions request: {error}",
-            model=model,
-            llm_provider=provider,
-        ) from error
-
-    resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
-    if resolved_api_base is None:
-        raise litellm.BadRequestError(
-            message=provider_config.missing_api_base_message(provider),
-            model=model,
-            llm_provider=provider,
-        )
-    resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
-    if resolved_api_key is None and provider_config.api_key_required:
-        raise litellm.AuthenticationError(
-            message=f"Missing API key for Decisions provider '{provider}'",
-            model=model,
-            llm_provider=provider,
-        )
-
-    ir_request: Final = _ir_request(request)
-    body: Final = provider_config.transform_decisions_request(
-        model=canonical_model, request=ir_request, custom_llm_provider=provider
-    )
-    if isinstance(body, UnsupportedDecisionsRequest):
-        raise litellm.BadRequestError(
-            message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}",
-            model=model,
-            llm_provider=provider,
-        )
-
-    logging_obj: Final = kwargs.get("litellm_logging_obj")
-    if isinstance(logging_obj, LiteLLMLoggingObj):
-        logging_obj.update_from_kwargs(
-            kwargs=dict(kwargs),
-            model=canonical_model,
-            litellm_params={
-                "litellm_call_id": kwargs.get("litellm_call_id"),
-                "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
-            },
-            custom_llm_provider=provider,
-        )
-    return _DecisionsCall(
-        model=canonical_model,
-        requested_model=model,
-        custom_llm_provider=provider,
-        provider_config=provider_config,
-        request=request,
-        ir_request=ir_request,
-        body=body,
-        api_base=resolved_api_base,
-        api_key=resolved_api_key,
-        logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
-        headers=extra_headers or {},
-        timeout=timeout,
-    )
-
-
-def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
-    formatted: Final = _formatted_response(response, call)
-    formatted.set_hidden_params(
+) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
         {
-            "model": f"{call.custom_llm_provider}/{call.model}",
-            "custom_llm_provider": call.custom_llm_provider,
-            "provider_response_model": f"{call.custom_llm_provider}/{call.model}",
-        }
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
     )
-    return formatted
-
-
-def _formatted_response(
-    response: DecisionsIRResponse, call: _DecisionsCall
-) -> DecisionsResponse | OpenAIDecisionResponse:
-    match call.request:
-        case DecisionsRequestBody():
-            return ir_to_systemone_response(response, call.ir_request)
-        case OpenAIDecisionRequestBody():
-            return ir_to_openai_response(response, call.ir_request, call.requested_model)
-        case _:
-            assert_never(call.request)
-
-
-def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception:
-    if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
-        provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
-        return litellm.APIConnectionError(
-            message=f"{provider_label} - {error.message}",
-            llm_provider=call.custom_llm_provider,
-            model=f"{call.custom_llm_provider}/{call.model}",
-        )
-    return litellm.exception_type(
-        model=f"{call.custom_llm_provider}/{call.model}",
-        custom_llm_provider=call.custom_llm_provider,
-        original_exception=error,
+    return prepare_call(
+        model=model,
+        fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),
+        safety_identifier=safety_identifier,
+        api_key=api_key,
+        api_base=api_base,
+        timeout=timeout,
+        custom_llm_provider=custom_llm_provider,
+        extra_headers=extra_headers,
+        kwargs={key: value for key, value in kwargs.items() if key not in _SYSTEMONE_FIELDS},
     )
 
 
 @client
 async def adecisions(
     model: str,
-    state: DecisionsJSON | None = None,
-    questions: DecisionsQuestions | None = None,
+    input: OpenAIDecisionInput | None = None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
+    safety_identifier: str | None = None,
     api_key: str | None = None,
     api_base: str | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: Mapping[str, str] | None = None,
-    input: OpenAIDecisionInput | None = None,
-    safety_identifier: str | None = None,
+    audio: DecisionsAudioClips | None = None,
+    videos: DecisionsVideoClips | None = None,
     **kwargs: object,
-) -> DecisionsResponse | OpenAIDecisionResponse:
-    call: Final = _prepare_call(
+) -> OpenAIDecisionResponse | DecisionsResponse:
+    call: Final = _prepare_decisions_call(
         model=model,
-        state=state,
-        questions=questions,
         decision_input=input,
+        questions=questions,
         safety_identifier=safety_identifier,
+        audio=audio,
+        videos=videos,
         api_key=api_key,
         api_base=api_base,
         timeout=timeout,
@@ -269,44 +129,31 @@ async def adecisions(
         extra_headers=extra_headers,
         kwargs=kwargs,
     )
-    try:
-        response: Final = await _HANDLER.adecisions(
-            model=call.model,
-            custom_llm_provider=call.custom_llm_provider,
-            logging_obj=call.logging_obj,
-            provider_config=call.provider_config,
-            request=call.ir_request,
-            body=call.body,
-            api_base=call.api_base,
-            api_key=call.api_key,
-            headers=call.headers,
-            timeout=call.timeout,
-        )
-    except Exception as error:
-        raise _map_upstream_exception(error, call) from error
-    return _format_response(response, call)
+    return await asend(call)
 
 
 @client
 def decisions(
     model: str,
-    state: DecisionsJSON | None = None,
-    questions: DecisionsQuestions | None = None,
+    input: OpenAIDecisionInput | None = None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
+    safety_identifier: str | None = None,
     api_key: str | None = None,
     api_base: str | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: Mapping[str, str] | None = None,
-    input: OpenAIDecisionInput | None = None,
-    safety_identifier: str | None = None,
+    audio: DecisionsAudioClips | None = None,
+    videos: DecisionsVideoClips | None = None,
     **kwargs: object,
-) -> DecisionsResponse | OpenAIDecisionResponse:
-    call: Final = _prepare_call(
+) -> OpenAIDecisionResponse | DecisionsResponse:
+    call: Final = _prepare_decisions_call(
         model=model,
-        state=state,
-        questions=questions,
         decision_input=input,
+        questions=questions,
         safety_identifier=safety_identifier,
+        audio=audio,
+        videos=videos,
         api_key=api_key,
         api_base=api_base,
         timeout=timeout,
@@ -314,22 +161,7 @@ def decisions(
         extra_headers=extra_headers,
         kwargs=kwargs,
     )
-    try:
-        response: Final = _HANDLER.decisions(
-            model=call.model,
-            custom_llm_provider=call.custom_llm_provider,
-            logging_obj=call.logging_obj,
-            provider_config=call.provider_config,
-            request=call.ir_request,
-            body=call.body,
-            api_base=call.api_base,
-            api_key=call.api_key,
-            headers=call.headers,
-            timeout=call.timeout,
-        )
-    except Exception as error:
-        raise _map_upstream_exception(error, call) from error
-    return _format_response(response, call)
+    return send(call)
 
 
 __all__ = ["adecisions", "decisions"]

litellm/images/main.py

@@ -24,7 +24,6 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.custom_llm import CustomLLM
 from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
 
 #################### Initialize provider clients ####################
@@ -505,10 +504,7 @@ def image_generation(
             )
         elif custom_llm_provider in litellm._custom_providers:  # Assume custom LLM provider
             # Get the Custom Handler
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -781,10 +777,7 @@ def image_edit(
 
         # Check for custom provider
         if custom_llm_provider in litellm._custom_providers:
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)

litellm/integrations/custom_guardrail.py

@@ -223,6 +223,7 @@ class CustomGuardrail(CustomLogger):
 
     records_own_guardrail_information: ClassVar[bool] = False
     logging_only_scope: LoggingOnlyScope | None
+    logging_only_continue_on_input_failure: bool
 
     stream_scope_default: GuardrailStreamScope = DEFAULT_GUARDRAIL_STREAM_SCOPE
     stream_scope_by_hook: tuple[tuple[str, GuardrailStreamScope], ...] = ()
@@ -305,6 +306,7 @@ class CustomGuardrail(CustomLogger):
         stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None))  # cast-ok: config
         self.apply_stream_scope(stream_scope_arg)
         self.logging_only_scope = None
+        self.logging_only_continue_on_input_failure = False
         if timeout is not None:
             self.timeout = timeout
 
@@ -1064,7 +1066,7 @@ class CustomGuardrail(CustomLogger):
         except Exception as e:
             if self.logging_only_scope == "output":
                 return None
-            if self.logging_only_scope == "both":
+            if self.logging_only_continue_on_input_failure:
                 verbose_logger.warning(
                     "Guardrail %s: logging_only request copy failed, skipping request scan: %s",
                     self.guardrail_name,
@@ -1092,7 +1094,7 @@ class CustomGuardrail(CustomLogger):
             "metadata": scratch_metadata,
         }
         if self.logging_only_scope != "output" and scratch_fields is not None:
-            if self.logging_only_scope == "both":
+            if self.logging_only_continue_on_input_failure:
                 try:
                     await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
                 except Exception as e:  # noqa: BLE001  # one direction's scan failure must not drop the other direction's verdict

litellm/integrations/gcs_bucket/gcs_bucket_base.py

@@ -18,7 +18,7 @@ from litellm.llms.custom_httpx.http_handler import (
     httpxSpecialProvider,
 )
 from litellm.types.integrations.gcs_bucket import *
-from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
+from litellm.types.utils import StandardLoggingPayload
 
 if TYPE_CHECKING:
     from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -137,49 +137,16 @@ class GCSBucketBase(CustomBatchLogger):
 
     async def get_gcs_logging_config(self, kwargs: dict[str, Any] | None = {}) -> GCSLoggingConfig:
         """
-        This function is used to get the GCS logging config for the GCS Bucket Logger.
-        It checks if the dynamic parameters are provided in the kwargs and uses them to get the GCS logging config.
-        If no dynamic parameters are provided, it uses the default values.
+        Returns the bucket and service account this logger was configured with.
         """
-        if kwargs is None:
-            kwargs = {}
-
-        standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
-            "standard_callback_dynamic_params", None
-        )
-
-        bucket_name: str
-        path_service_account: str | None
-        if standard_callback_dynamic_params is not None:
-            verbose_logger.debug("Using dynamic GCS logging")
-            verbose_logger.debug("standard_callback_dynamic_params: %s", standard_callback_dynamic_params)
-
-            _bucket_name: str | None = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME
-            _path_service_account: Final[str | None] = (
-                standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json
+        if self.BUCKET_NAME is None:
+            raise ValueError(
+                "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
             )
-
-            if _bucket_name is None:
-                raise ValueError(
-                    "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
-                )
-            bucket_name = _bucket_name
-            path_service_account = _path_service_account
-            vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
-        else:
-            # If no dynamic parameters, use the default instance
-            if self.BUCKET_NAME is None:
-                raise ValueError(
-                    "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
-                )
-            bucket_name = self.BUCKET_NAME
-            path_service_account = self.path_service_account_json
-            vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
-
         return GCSLoggingConfig(
-            bucket_name=bucket_name,
-            vertex_instance=vertex_instance,
-            path_service_account=path_service_account,
+            bucket_name=self.BUCKET_NAME,
+            vertex_instance=await self.get_or_create_vertex_instance(credentials=self.path_service_account_json),
+            path_service_account=self.path_service_account_json,
         )
 
     async def get_or_create_vertex_instance(self, credentials: str | None) -> VertexBase:

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -219,7 +219,7 @@ class GitLabTemplateManager:
                 elif value.replace(".", "").isdigit():
                     try:
                         result[key] = float(value)
-                    except Exception:
+                    except ValueError:
                         result[key] = value
                 else:
                     result[key] = value.strip("\"'")

litellm/integrations/otel/mount.py

@@ -104,8 +104,8 @@ def instrument_fastapi_app(app: "FastAPI") -> None:
 
     Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
     is unavailable. This MUST be called at app-creation time — once the lifespan
-    runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
-    middleware after an application has started".
+    runs, the middleware stack is frozen and middleware added later never runs, so
+    a late call is skipped without registering anything.
 
     No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
     ``ProxyTracerProvider``; the proxy publishes the real provider as the global
@@ -115,6 +115,9 @@ def instrument_fastapi_app(app: "FastAPI") -> None:
     try:
         if not is_otel_v2_enabled():
             return
+        if app.middleware_stack is not None:
+            verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
+            return
 
         # Lazy: only the V2-enabled path needs the optional
         # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

litellm/integrations/otel/plumbing/otlp_json.py

@@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
 
 import base64
 import json
+import logging
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
 from typing import Final, TypeAlias
@@ -13,9 +14,13 @@ from typing import Final, TypeAlias
 import requests
 from google.protobuf.json_format import MessageToDict
 from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
+from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
 
+_logger: Final = logging.getLogger(__name__)
 JSON_CONTENT_TYPE: Final = "application/json"
 _HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
 
@@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
     def __init__(
         self,
         endpoint: str | None,
-        headers: dict[str, str],  # mutable-ok: SDK __init__ takes Dict
+        headers: Mapping[str, str],
         certificate_file: str | None = None,
         session: "requests.Session | None" = None,
+        *,
+        max_request_size: int | None = None,
     ) -> None:
-        super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
-        self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
+        super().__init__(
+            endpoint=endpoint,
+            headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
+            certificate_file=certificate_file,
+            session=session,
+            max_request_size=max_request_size,
+        )
 
-    def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
-        return encode_spans_json(spans)
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        if self._shutdown:
+            _logger.warning("Exporter already shutdown, ignoring batch")
+            return SpanExportResult.FAILURE
+
+        with self._metrics.export_operation(len(spans)) as result:
+            try:
+                serialized_data: Final = encode_spans_json(spans)
+            except Exception as error:  # noqa: BLE001  # SDK export contract reports encoding failures as results
+                _logger.error("Failed to encode span batch: %s", error)
+                result.error = error
+                return SpanExportResult.FAILURE
+
+            if 0 < self._max_request_size < len(serialized_data):
+                result.error = RequestPayloadTooLargeError(
+                    f"Serialized span request size {len(serialized_data)} bytes "
+                    f"exceeds max_request_size {self._max_request_size} bytes."
+                )
+                _logger.warning("Dropping span batch: %s", result.error)
+                return SpanExportResult.FAILURE
+
+            export_result: Final = self._client.export(serialized_data)
+            if not export_result.success:
+                result.error = export_result.error
+                result.error_attrs = (
+                    {HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
+                    if export_result.status_code is not None
+                    else None
+                )
+                return SpanExportResult.FAILURE
+
+        return SpanExportResult.SUCCESS

litellm/integrations/otel/presets/agentops.py

@@ -76,6 +76,7 @@ def agentops_preset(
 
 def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
     """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
+    import requests
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
@@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
         """
 
         def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
-            super().__init__(endpoint=endpoint)
+            self._agentops_session = requests.Session()
+            super().__init__(endpoint=endpoint, session=self._agentops_session)
             self._agentops_api_key = api_key
             self._auth_resolved = False
 
@@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
             try:
                 token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
                 if token:
-                    # ``_session`` is the requests.Session the base exporter
-                    # POSTs through; updating its Authorization header is how the
-                    # minted JWT reaches every subsequent export.
-                    self._session.headers["Authorization"] = f"Bearer {token}"
+                    self._agentops_session.headers["Authorization"] = f"Bearer {token}"
             except Exception as e:
                 verbose_logger.debug("AgentOps JWT fetch failed: %s", e)
 

litellm/litellm_core_utils/core_helpers.py

@@ -3,9 +3,9 @@
 import copy
 import logging
 import re
-from collections.abc import Collection, Iterable, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable
 
 import httpx
 from pydantic import TypeAdapter, ValidationError
@@ -52,6 +52,28 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
 _DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
 
 
+def as_str_mapping(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, Mapping) else None
+
+
+@runtime_checkable
+class _SupportsIter(Protocol):
+    def __iter__(self) -> Iterator[object]: ...
+
+
+@runtime_checkable
+class _SupportsGetItem(Protocol):
+    def __getitem__(self, index: int, /) -> object: ...
+
+
+def iter_items(value: object) -> Iterable[object]:
+    if isinstance(value, _SupportsIter):
+        return value
+    if isinstance(value, _SupportsGetItem):
+        return iter(value)
+    raise TypeError(f"'{type(value).__name__}' object is not iterable")
+
+
 def normalize_drop_params(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value

litellm/litellm_core_utils/get_blog_posts.py

@@ -8,7 +8,6 @@ Disable remote fetching entirely:
     export LITELLM_LOCAL_BLOG_POSTS=True
 """
 
-import json
 import os
 import time
 import xml.etree.ElementTree as ET
@@ -17,22 +16,53 @@ from importlib.resources import files
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict
 
 from litellm import verbose_logger
 from litellm.types.llms.base import LiteLLMBaseModel
 
 BLOG_POSTS_TTL_SECONDS: Final[int] = 3600  # 1 hour
+BLOG_POSTS_MAX: Final[int] = 50
 
 
 class BlogPost(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
     title: str
     description: str
     date: str
     url: str
+    tags: tuple[str, ...] = ()
 
 
 class BlogPostsResponse(LiteLLMBaseModel):
-    posts: list[BlogPost]
+    model_config = ConfigDict(frozen=True)
+
+    posts: tuple[BlogPost, ...]
+
+
+def _pub_date(pub_date_el: ET.Element | None) -> str:
+    if pub_date_el is None or not pub_date_el.text:
+        return ""
+    try:
+        return parsedate_to_datetime(pub_date_el.text).strftime("%Y-%m-%d")
+    except Exception:
+        return pub_date_el.text
+
+
+def _parse_item(item: ET.Element) -> BlogPost | None:
+    title_el: Final = item.find("title")
+    link_el: Final = item.find("link")
+    if title_el is None or link_el is None:
+        return None
+    desc_el: Final = item.find("description")
+    return BlogPost(
+        title=title_el.text or "",
+        description=(desc_el.text or "") if desc_el is not None else "",
+        date=_pub_date(item.find("pubDate")),
+        url=link_el.text or "",
+        tags=tuple(category.text for category in item.findall("category") if category.text),
+    )
 
 
 class GetBlogPosts:
@@ -40,19 +70,20 @@ class GetBlogPosts:
     Fetches, validates, and caches LiteLLM blog posts.
 
     - Fetches RSS feed from docs site with a 5-second timeout
-    - Parses the XML and extracts the latest blog post
+    - Parses the XML and extracts up to BLOG_POSTS_MAX blog posts, with their RSS categories as tags
     - Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour)
     - Falls back to the bundled local backup on any failure
     """
 
-    _cached_posts: list[dict[str, str]] | None = None
+    _cached_posts: tuple[BlogPost, ...] | None = None
     _last_fetch_time: float = 0.0
 
     @staticmethod
-    def load_local_blog_posts() -> list[dict[str, str]]:
+    def load_local_blog_posts() -> tuple[BlogPost, ...]:
         """Load the bundled local backup blog posts."""
-        content: Final = json.loads(files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8"))
-        return content.get("posts", [])
+        return BlogPostsResponse.model_validate_json(
+            files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8")
+        ).posts
 
     @staticmethod
     def fetch_rss_feed(url: str, timeout: int = 5) -> str:
@@ -66,54 +97,24 @@ class GetBlogPosts:
         return response.text
 
     @staticmethod
-    def parse_rss_to_posts(xml_text: str, max_posts: int = 1) -> list[dict[str, str]]:
+    def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
         """
-        Parse RSS XML and return a list of blog post dicts.
+        Parse RSS XML into blog posts.
 
-        Extracts title, description, date (YYYY-MM-DD), and url from each <item>.
+        Extracts title, description, date (YYYY-MM-DD), url and the <category> tags from each <item>.
         """
         root: Final = ET.fromstring(xml_text)
         channel: Final = root.find("channel")
         if channel is None:
             raise ValueError("RSS feed missing <channel> element")
 
-        posts: Final[list[dict[str, str]]] = []
-        for item in channel.findall("item"):
-            if len(posts) >= max_posts:
-                break
-
-            title_el = item.find("title")
-            link_el = item.find("link")
-            desc_el = item.find("description")
-            pub_date_el = item.find("pubDate")
-
-            if title_el is None or link_el is None:
-                continue
-
-            # Parse RFC 2822 date to YYYY-MM-DD
-            date_str = ""
-            if pub_date_el is not None and pub_date_el.text:
-                try:
-                    dt = parsedate_to_datetime(pub_date_el.text)
-                    date_str = dt.strftime("%Y-%m-%d")
-                except Exception:
-                    date_str = pub_date_el.text
-
-            posts.append(
-                {
-                    "title": title_el.text or "",
-                    "description": desc_el.text or "" if desc_el is not None else "",
-                    "date": date_str,
-                    "url": link_el.text or "",
-                }
-            )
-
-        return posts
+        parsed: Final = (_parse_item(item) for item in channel.findall("item"))
+        return tuple(post for post in parsed if post is not None)[:max_posts]
 
     @staticmethod
-    def validate_blog_posts(posts: list[dict[str, str]]) -> bool:
-        """Return True if posts is a non-empty list."""
-        if not isinstance(posts, list) or len(posts) == 0:
+    def validate_blog_posts(posts: tuple[BlogPost, ...]) -> bool:
+        """Return True if there is at least one post."""
+        if len(posts) == 0:
             verbose_logger.warning(
                 "LiteLLM: Parsed RSS feed has no valid posts. Falling back to local backup.",
             )
@@ -121,9 +122,9 @@ class GetBlogPosts:
         return True
 
     @classmethod
-    def get_blog_posts(cls, url: str) -> list[dict[str, str]]:
+    def get_blog_posts(cls, url: str) -> tuple[BlogPost, ...]:
         """
-        Return the blog posts list.
+        Return the blog posts.
 
         Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
         Fetches from ``url`` otherwise, falling back to local backup on failure.
@@ -155,6 +156,6 @@ class GetBlogPosts:
         return posts
 
 
-def get_blog_posts(url: str) -> list[dict[str, str]]:
-    """Public entry point — returns the blog posts list."""
+def get_blog_posts(url: str) -> tuple[BlogPost, ...]:
+    """Public entry point, returns the blog posts."""
     return GetBlogPosts.get_blog_posts(url=url)

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -1,4 +1,4 @@
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import Final
 
 from litellm.types.utils import ProviderSpecificHeader
@@ -9,7 +9,7 @@ class ProviderSpecificHeaderUtils:
     def get_provider_specific_headers(
         provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
         custom_llm_provider: str | None,
-    ) -> dict:
+    ) -> Mapping[str, str]:
         """
         Get the provider specific headers for the given custom llm provider.
 

litellm/litellm_core_utils/health_check_helpers.py

@@ -3,7 +3,7 @@ Helper functions for health check calls.
 """
 
 import base64
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
 from typing import TYPE_CHECKING, Final, Literal
 
 from litellm.llms.base_llm.ocr.transformation import DocumentType
@@ -56,6 +56,21 @@ def get_image_file_for_health_check() -> bytes:
     return base64.b64decode(TEST_IMAGE_BASE64)
 
 
+def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]:
+    import litellm
+    from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+    from litellm.utils import ProviderConfigManager
+
+    provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None)
+    config: Final = (
+        None
+        if provider is None
+        else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider)
+    )
+    questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions
+    return {name: dict(question) for name, question in questions.items()}
+
+
 def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
     native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load()
     if native is None:
@@ -199,7 +214,7 @@ class HealthCheckHelpers:
             "responses",
             "anthropic_messages",
             "ocr",
-            "evaluation",
+            "decisions",
         ],
         Callable,
     ]:
@@ -296,11 +311,13 @@ class HealthCheckHelpers:
                 **_filter_model_params(model_params=model_params),
                 document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
             ),
-            "evaluation": lambda: litellm.adecisions(
+            "decisions": lambda: litellm.asystemone(
                 **DECISIONS_CALL_PARAMS.validate_python(
                     {
                         "state": prompt or "health check",
-                        "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
+                        "questions": _decisions_health_check_questions(
+                            model=model, custom_llm_provider=custom_llm_provider
+                        ),
                         **_filter_model_params(model_params=model_params),
                     }
                 )

litellm/litellm_core_utils/litellm_logging.py

@@ -18,7 +18,7 @@ from types import MappingProxyType, TracebackType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 
 from httpx import Response
-from pydantic import BaseModel, JsonValue
+from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
 from litellm import _custom_logger_compatible_callbacks_literal
@@ -41,6 +41,7 @@ from litellm.constants import (
     DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
+    LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
     PROVIDER_REQUEST_ID_HEADERS,
     REDACTED_BY_LITELLM,
 )
@@ -385,6 +386,8 @@ class RawRequestCaptured(Exception):
     pass
 
 
+_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
+
 _DEPLOYMENT_PRICING_KEYS: Final = (
     "input_cost_per_token",
     "output_cost_per_token",
@@ -496,7 +499,7 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None:
         return None
 
 
-def _resolve_vertex_location_for_cost(
+def resolve_vertex_location_for_cost(
     custom_llm_provider: str | None,
     litellm_params: Mapping[str, object] | None,
     optional_params: Mapping[str, object] | None,
@@ -757,6 +760,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.defer_async_logging: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
+        self._async_success_enqueued: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -1445,15 +1449,20 @@ class Logging(LiteLLMLoggingBaseClass):
                         "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
                     )
 
-            self.model_call_details["api_call_start_time"] = datetime.datetime.now()
-            # Set-once first provider-handoff instant. api_call_start_time
-            # is overwritten on every retry, so it can't measure one-time
-            # preprocessing; pinning the first attempt excludes retry loops
-            # + backoff. Logging object only — must NOT go into
-            # litellm_params["metadata"] (caller request metadata, typed
-            # Dict[str, str], echoed downstream; a datetime breaks it).
-            if self.model_call_details.get("first_api_call_start_time") is None:
-                self.model_call_details["first_api_call_start_time"] = self.model_call_details["api_call_start_time"]
+            # A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks
+            # without any provider handoff, so it must not stamp the handoff instants.
+            if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details:
+                self.model_call_details["api_call_start_time"] = datetime.datetime.now()
+                # Set-once first provider-handoff instant. api_call_start_time
+                # is overwritten on every retry, so it can't measure one-time
+                # preprocessing; pinning the first attempt excludes retry loops
+                # + backoff. Logging object only — must NOT go into
+                # litellm_params["metadata"] (caller request metadata, typed
+                # Dict[str, str], echoed downstream; a datetime breaks it).
+                if self.model_call_details.get("first_api_call_start_time") is None:
+                    self.model_call_details["first_api_call_start_time"] = self.model_call_details[
+                        "api_call_start_time"
+                    ]
             # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
             callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
             for callback in callbacks:
@@ -1891,7 +1900,7 @@ class Logging(LiteLLMLoggingBaseClass):
             router_model_id = self.get_router_model_id()
 
         ## RESPONSE COST ##
-        custom_pricing: Final = self._custom_pricing_for(priced_result)
+        custom_pricing: Final = self.custom_pricing_for(priced_result)
 
         prompt: Final = self._prompt_for_cost_calculation()
 
@@ -1927,7 +1936,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     if hasattr(self, "litellm_params") and self.litellm_params
                     else None
                 ),
-                "vertex_location": _resolve_vertex_location_for_cost(
+                "vertex_location": resolve_vertex_location_for_cost(
                     custom_llm_provider=cost_custom_llm_provider,
                     litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None),
                     optional_params=self.optional_params,
@@ -2041,7 +2050,7 @@ class Logging(LiteLLMLoggingBaseClass):
             model=model,
             completion_response=result,
             custom_llm_provider=custom_llm_provider,
-            custom_pricing=self._custom_pricing_for(result),
+            custom_pricing=self.custom_pricing_for(result),
             base_model=get_base_model_from_metadata(model_call_details=self.model_call_details),
             router_model_id=router_model_id or self.get_router_model_id(),
             region_name=_resolve_mantle_region_for_cost(
@@ -2066,20 +2075,23 @@ class Logging(LiteLLMLoggingBaseClass):
             usage=usage,
         )
 
-    def _custom_pricing_for(self, result: object) -> bool:
-        litellm_params: Final = getattr(self, "litellm_params", None)
+    def spilled_over_ptu_for(self, result: object) -> bool:
         result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
         additional_headers: Final = (
             result_hidden_params.get("additional_headers")
             if isinstance(result_hidden_params, dict)
             else getattr(result_hidden_params, "additional_headers", None)
         )
-        spilled_over: Final = is_spilled_over_ptu_request(
-            model_info=_deployment_model_info(litellm_params),
+        return is_spilled_over_ptu_request(
+            model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
             response_headers=self.model_call_details.get("response_headers"),
             additional_headers=additional_headers,
         )
-        return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
+
+    def custom_pricing_for(self, result: object) -> bool:
+        if self.spilled_over_ptu_for(result):
+            return False
+        return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
 
     def _prompt_for_cost_calculation(self) -> str:
         """
@@ -2288,6 +2300,16 @@ class Logging(LiteLLMLoggingBaseClass):
         except Exception:
             return True
 
+    @property
+    def async_success_log_enqueued(self) -> bool:
+        """True once the success log task was actually created, not merely claimed, so a later failure raised
+        around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
+        time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
+        return self._async_success_enqueued
+
+    def mark_async_success_log_enqueued(self) -> None:
+        self._async_success_enqueued = True
+
     def claim_async_success_log(self) -> bool:
         """One async success log per request. The innermost @client wrapper always exits first,
         since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
@@ -3319,6 +3341,14 @@ class Logging(LiteLLMLoggingBaseClass):
                     model_name=self.get_deployment_model_for_cost(),
                     litellm_params=self.litellm_params,
                     model_info=self.get_router_deployment_model_info(),
+                    vertex_location=resolve_vertex_location_for_cost(
+                        custom_llm_provider=self.custom_llm_provider,
+                        litellm_params=_LITELLM_PARAMS.validate_python(
+                            self.litellm_params  # pyright: ignore[reportUnknownMemberType]  # Logging.litellm_params is untyped
+                        ),
+                        optional_params=None,
+                        model=self.get_deployment_model_for_cost() or self.model,
+                    ),
                 )
 
                 set_hidden_param(result, "response_cost", batch_result.cost)
@@ -6023,6 +6053,7 @@ class StandardLoggingPayloadSetup:
         custom_llm_provider: str | None,
         init_response_obj: object,
         api_base: str | None = None,
+        router_model_id: str | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -6030,12 +6061,23 @@ class StandardLoggingPayloadSetup:
             base_model=base_model,
             custom_pricing=custom_pricing,
         )
+        pricing_model_name: Final = (
+            select_model_name_for_cost_calc(
+                model=base_model if custom_pricing else None,
+                completion_response=init_response_obj,
+                base_model=base_model,
+                custom_pricing=custom_pricing,
+                router_model_id=router_model_id,
+            )
+            if router_model_id is not None
+            else model_cost_name
+        )
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
             try:
                 _model_cost_information: Final = litellm.get_model_info(
-                    model=model_cost_name,
+                    model=pricing_model_name or model_cost_name,
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
                 )
@@ -6436,19 +6478,17 @@ class StandardLoggingPayloadSetup:
         litellm_metadata: Final = cast(  # cast-ok: request metadata is caller-provided
             Mapping[str, object], litellm_params.get("litellm_metadata") or {}
         )
-        if metadata.get("tags", []):
-            request_tags = cast(list[str], metadata.get("tags", [])).copy()  # cast-ok: tags are caller-provided
-        elif litellm_metadata.get("tags", []):
-            request_tags = cast(list[str], litellm_metadata.get("tags", [])).copy()  # cast-ok: tags are caller-provided
-        else:
-            request_tags = []
+        request_tags: Final = next(
+            (
+                cast(list[str], tags)  # cast-ok: tags are caller-provided
+                for tags in (metadata.get("tags"), litellm_metadata.get("tags"))
+                if isinstance(tags, list) and tags
+            ),
+            [],
+        )
         user_agent_tags: Final = StandardLoggingPayloadSetup._get_user_agent_tags(proxy_server_request)
         additional_header_tags: Final = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request)
-        if user_agent_tags is not None:
-            request_tags.extend(user_agent_tags)
-        if additional_header_tags is not None:
-            request_tags.extend(additional_header_tags)
-        return request_tags
+        return [*request_tags, *(user_agent_tags or ()), *(additional_header_tags or ())]
 
     _get_request_tags = get_request_tags
 
@@ -6686,13 +6726,16 @@ def get_standard_logging_object_payload(
             )
 
         ## Get model cost information ##
-        base_model = get_base_model_from_metadata(model_call_details=kwargs)
-        # The router overrides completion_response.model to the model-group alias before
-        # this payload is built, so cost-map lookup via that alias always misses.
-        # Fall back to the actual deployment model set by the router in metadata.
-        if base_model is None:
-            base_model = metadata.get("deployment")
-        custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params)
+        configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)
+        spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
+        # The router overrides completion_response.model to the model-group alias before this payload is
+        # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
+        # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
+        # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
+        base_model: Final = (
+            metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
+        )
+        custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
         raw_response_cost: Final = kwargs.get("response_cost")
         llm_response_cost: Final[float] = raw_response_cost or 0.0
         guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
@@ -6714,6 +6757,7 @@ def get_standard_logging_object_payload(
             custom_llm_provider=kwargs.get("custom_llm_provider"),
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
+            router_model_id=logging_obj.get_router_model_id(),
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -285,7 +285,7 @@ def _parse_token_threshold(threshold: str) -> float:
 
 
 def _parse_above_token_threshold(key: str) -> float:
-    return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0])
+    return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0])
 
 
 def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool:
@@ -681,10 +681,12 @@ def _get_token_base_cost(
     # Optimization: collect threshold keys first to avoid sorting all model_info keys.
     # Standard thresholds and thresholds suffixed for this request's service tier both count.
     tier_key_suffix: Final = get_service_tier_cost_key("", service_tier)
+    threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES)
     threshold_keys: Final = [
         k
         for k in model_info
-        if k.startswith("input_cost_per_token_above_")
+        if k.startswith(threshold_prefixes)
+        and "_tokens" in k
         and (
             not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix))
         )
@@ -692,12 +694,12 @@ def _get_token_base_cost(
 
     # Only sort the threshold keys (typically 1-2 keys instead of 66+)
     threshold: float | None = None
-    for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True):
+    for key in sorted(threshold_keys, key=_parse_above_token_threshold):
         value = model_info.get(key)
         if value is not None:
             try:
                 # Handle both formats: _above_128k_tokens and _above_128_tokens
-                threshold_str = key.split("_above_")[1].split("_tokens")[0]
+                threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0]
                 threshold = _parse_above_token_threshold(key)
                 if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive):
                     # Prefer a service_tier-specific above-threshold key when available,
@@ -710,7 +712,7 @@ def _get_token_base_cost(
                             service_tier,
                         )
                         if service_tier
-                        else key
+                        else f"input_cost_per_token_above_{threshold_str}_tokens"
                     )
                     prompt_base_cost = cast(
                         float,
@@ -766,8 +768,6 @@ def _get_token_base_cost(
                     )
 
                     cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
-
-                    break
             except (IndexError, ValueError):
                 continue
             except Exception:
@@ -1086,6 +1086,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
     )
 
 
+def _modality_token_cost(
+    model_info: ModelInfo,
+    modality_cost_key: str,
+    tokens: int,
+    prompt_base_cost: float,
+    service_tier: str | None,
+) -> float:
+    """
+    Audio, image and video prompt tokens bill at their modality rate when the model has one.
+    Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
+    `_get_token_base_cost` has already resolved for the service tier and long-context tier.
+    """
+    tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
+    cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
+    if model_info.get(cost_key) is None:
+        return float(tokens) * prompt_base_cost
+    return calculate_cost_component(model_info, cost_key, tokens)
+
+
 def _calculate_input_cost(
     prompt_tokens_details: PromptTokensDetailsResult,
     model_info: ModelInfo,
@@ -1097,8 +1116,14 @@ def _calculate_input_cost(
 ) -> float:
     """
     Calculates the input cost for a given model, prompt tokens, and completion tokens.
+
+    A model priced per character (`input_cost_per_character`) bills the reported character count
+    in place of its text tokens, so the same prompt is never charged under both units.
     """
-    prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
+    bills_characters: Final = (
+        prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
+    )
+    prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
 
     ### CACHE READ COST - Now uses tiered pricing
     cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
@@ -1116,28 +1141,37 @@ def _calculate_input_cost(
     if prompt_tokens_details["audio_tokens"] and not (
         prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
     ):
-        audio_cost_key: Final = get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
-        prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_audio_token",
+            prompt_tokens_details["audio_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### IMAGE TOKEN COST
     if prompt_tokens_details["image_tokens"] and not (
         prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
     ):
-        # For image token costs:
-        # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
-        image_token_cost_key = "input_cost_per_image_token"
-        if model_info.get(image_token_cost_key) is None:
-            image_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_image_token",
+            prompt_tokens_details["image_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### VIDEO TOKEN COST
     if prompt_tokens_details["video_tokens"] and not (
         prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
     ):
-        video_token_cost_key = "input_cost_per_video_token"
-        if model_info.get(video_token_cost_key) is None:
-            video_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_video_token",
+            prompt_tokens_details["video_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### CACHE WRITING COST - Now uses tiered pricing
     if (
@@ -1654,20 +1688,23 @@ def get_billed_token_rates(
     vertex_location: str | None = None,
     current_time: datetime | None = None,
     custom_cost_per_token: CostPerToken | None = None,
+    model_info: ModelInfo | None = None,
 ) -> BilledTokenRates | None:
     """Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type
     breakdown resolve them. None when the model's pricing cannot be resolved."""
     if custom_cost_per_token is not None:
         return _custom_pricing_rates(custom_cost_per_token)
     try:
-        model_info: Final = apply_provider_cache_read_default(
-            get_model_info(model=model, custom_llm_provider=custom_llm_provider),
+        resolved_model_info: Final = apply_provider_cache_read_default(
+            model_info
+            if model_info is not None
+            else get_model_info(model=model, custom_llm_provider=custom_llm_provider),
             custom_llm_provider,
         )
     except Exception:  # noqa: BLE001  # get_model_info raises a bare Exception for an unmapped model: no rates
         return None
     return _cost_map_billed_rates(
-        model_info=model_info,
+        model_info=resolved_model_info,
         usage=usage,
         custom_llm_provider=custom_llm_provider,
         service_tier=service_tier,
@@ -1686,6 +1723,7 @@ def get_token_type_cost_breakdown(
     vertex_location: str | None = None,
     current_time: datetime | None = None,
     custom_cost_per_token: CostPerToken | None = None,
+    model_info: ModelInfo | None = None,
 ) -> TokenTypeCostBreakdown:
     """
     Provider-agnostic cost of reasoning and cache tokens, derived from the usage
@@ -1710,6 +1748,7 @@ def get_token_type_cost_breakdown(
         vertex_location=vertex_location,
         current_time=current_time,
         custom_cost_per_token=custom_cost_per_token,
+        model_info=model_info,
     )
     if rates is None:
         return TokenTypeCostBreakdown(0.0, 0.0, 0.0)

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -4,6 +4,7 @@ from functools import reduce
 from typing import Final
 
 import httpx
+from pydantic import TypeAdapter, ValidationError
 
 from litellm.constants import LITELLM_DETAILED_TIMING
 from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
@@ -101,6 +102,32 @@ def response_timing_metrics(
     return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
 
 
+_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
+
+
+def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
+    try:
+        return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
+    except ValidationError:
+        return {}
+
+
+def _existing_provider_response_model(result: object) -> str | None:
+    """The pricing model a response already carries, like the decisions path sets, wins over the response model."""
+    hidden_params: Final[object] = getattr(result, "_hidden_params", None)
+    carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
+        hidden_params, "provider_response_model", None
+    )
+    return carried if isinstance(carried, str) and carried else None
+
+
+def _provider_response_model(result: object) -> str | None:
+    """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
+    client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
+    model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
+    return model if isinstance(model, str) and model else None
+
+
 class ResponseMetadata:
     """
     Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@@ -133,6 +160,8 @@ class ResponseMetadata:
                 preserve_litellm_internal_headers=True,
             ),
             "litellm_model_name": model,
+            "provider_response_model": _existing_provider_response_model(self.result)
+            or _provider_response_model(self.result),
         }
         self._update_hidden_params(new_params)
 

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -2100,10 +2100,8 @@ def is_unsignable_thinking_block(block: object) -> bool:
     Anthropic verifies the signature cryptographically, so a block with a null,
     empty, or missing signature (e.g. from an open-source reasoning model) is
     rejected with a 400, and so is a block whose signature or data carries
-    another provider's encrypted reasoning. It also rejects a `thinking` block
-    whose text is empty or whitespace-only ("each thinking block must contain
-    thinking"), regardless of signature, e.g. when a `thinking_blocks` history
-    item from a non-Anthropic reasoning provider is replayed through this path.
+    another provider's encrypted reasoning. A signed block is kept whatever its
+    text, since ``display: omitted`` returns signed blocks with empty ``thinking``.
     `redacted_thinking` blocks carry no signature and are always kept.
     """
     if is_encrypted_reasoning_block(block):
@@ -2114,10 +2112,7 @@ def is_unsignable_thinking_block(block: object) -> bool:
     if mapping.get("type") != "thinking":
         return False
     signature: Final = mapping.get("signature")
-    if not (isinstance(signature, str) and len(signature) > 0):
-        return True
-    thinking_text: Final = mapping.get("thinking")
-    return not (isinstance(thinking_text, str) and len(thinking_text.strip()) > 0)
+    return not (isinstance(signature, str) and len(signature) > 0)
 
 
 def strip_encrypted_reasoning_from_messages(

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -2774,13 +2774,8 @@ def anthropic_messages_pt(
                         if not isinstance(m, dict):
                             continue
                         # handle thinking blocks
-                        thinking_block = cast(str, m.get("thinking", ""))
                         text_block = cast(str, m.get("text", ""))
-                        if (
-                            m.get("type", "") == "thinking"
-                            and len(thinking_block) > 0
-                            and not is_unsignable_thinking_block(m)
-                        ):  # don't pass empty text blocks. anthropic api raises errors.
+                        if m.get("type", "") == "thinking" and not is_unsignable_thinking_block(m):
                             anthropic_message: ChatCompletionThinkingBlock | AnthropicMessagesTextParam = cast(
                                 ChatCompletionThinkingBlock, m
                             )
@@ -5781,3 +5776,25 @@ def resolve_structured_messages(
         if structured:
             return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
     return None
+
+
+def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
+    tool_type: Final = tool.get("type")
+    return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
+
+
+def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+    from litellm.responses.litellm_completion_transformation.transformation import (
+        LiteLLMCompletionResponsesConfig,
+    )
+
+    if is_flat_responses_tool(tool):
+        chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
+            [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in a flat Responses tool shape
+        )
+        return tuple(chat_tools)
+    anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
+        [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in an Anthropic or chat tool shape
+    )
+    return tuple(anthropic_tools)

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -4,12 +4,14 @@ Helper functions to handle images passed in messages
 
 import asyncio
 import base64
+import time
 from collections.abc import Callable, Iterable, Mapping
 from dataclasses import dataclass
 from itertools import chain
 from types import MappingProxyType
 from typing import Final
 
+import httpx
 from httpx import Response
 
 import litellm
@@ -22,6 +24,7 @@ from litellm.types.llms.openai import AllMessageValues
 
 MAX_IMGS_IN_MEMORY: Final = 10
 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
+_FETCH_ATTEMPTS: Final = 3
 
 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
 
@@ -73,6 +76,13 @@ def _process_image_response(response: Response, url: str) -> str:
     return result
 
 
+def _attempt_timeouts(timeout: float | httpx.Timeout | None) -> Iterable[float | httpx.Timeout | None]:
+    if timeout is None or isinstance(timeout, httpx.Timeout):
+        return (timeout,) * _FETCH_ATTEMPTS
+    deadline: Final = time.monotonic() + timeout
+    return (remaining for _ in range(_FETCH_ATTEMPTS) if (remaining := deadline - time.monotonic()) > 0)
+
+
 def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchError":
     verbose_logger.warning("Image fetch of %s rejected before any request went out: %s", url, verdict)
     return litellm.ImageFetchError(
@@ -81,7 +91,7 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr
     )
 
 
-async def async_convert_url_to_base64(url: str) -> str:
+async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
     if url.startswith("data:") and ";base64," in url:
         return url
 
@@ -96,9 +106,9 @@ async def async_convert_url_to_base64(url: str) -> str:
         return cached_result
 
     client: Final = litellm.module_level_aclient
-    for _ in range(3):
+    for attempt_timeout in _attempt_timeouts(timeout):
         try:
-            response = await async_safe_get(client, url)
+            response = await async_safe_get(client, url, timeout=attempt_timeout)
             return _process_image_response(response, url)
         except litellm.ImageFetchError:
             raise
@@ -109,7 +119,7 @@ async def async_convert_url_to_base64(url: str) -> str:
     raise litellm.ImageFetchError(f"Error: Unable to fetch image from URL after 3 attempts. url={url}")
 
 
-def convert_url_to_base64(url: str) -> str:
+def convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
     if url.startswith("data:") and ";base64," in url:
         return url
 
@@ -124,9 +134,9 @@ def convert_url_to_base64(url: str) -> str:
         return cached_result
 
     client: Final = litellm.module_level_client
-    for _ in range(3):
+    for attempt_timeout in _attempt_timeouts(timeout):
         try:
-            response = safe_get(client, url)
+            response = safe_get(client, url, timeout=attempt_timeout)
             return _process_image_response(response, url)
         except litellm.ImageFetchError:
             raise
@@ -286,14 +296,16 @@ def _inline_message(
     return inlined_message  # pyright: ignore[reportReturnType]  # the same message with its remote parts inlined
 
 
-async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore) -> str:
+async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
     async with in_flight:
-        return await async_convert_url_to_base64(url)
+        return await async_convert_url_to_base64(url, timeout=timeout)
 
 
-async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]:
+async def async_convert_urls_to_base64(
+    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
+) -> tuple[str, ...]:
     in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
-    fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight)) for url in remote_urls)
+    fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
     try:
         return tuple(await asyncio.gather(*fetches))
     except BaseException:
@@ -329,6 +341,6 @@ async def async_inline_remote_media(
     remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
     if not remote_urls:
         return messages
-    data_urls: Final = await _fetch_data_urls(remote_urls)
+    data_urls: Final = await async_convert_urls_to_base64(remote_urls)
     inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
     return [_inline_message(message, inlined, should_inline) for message in messages]

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -265,8 +265,7 @@ def _assistant_part_renders(part: object) -> bool:
     """A text part always renders: the converter pads empty text with a placeholder."""
     part_type: Final = message_field(part, "type")
     if part_type == "thinking":
-        thinking: Final = message_field(part, "thinking")
-        return isinstance(thinking, str) and bool(thinking) and _thinking_block_renders(part)
+        return _thinking_block_renders(part)
     return part_type in _RENDERED_ASSISTANT_PART_TYPES or (
         isinstance(part_type, str) and part_type.endswith("_tool_result")
     )

litellm/litellm_core_utils/request_timeout_resolver.py

@@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
 
 from __future__ import annotations
 
-from typing import Final
+from collections.abc import Mapping
+from typing import Final, Literal, Protocol
+
+from pydantic import TypeAdapter
 
 from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
 
+_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
+
 
 def get_configured_request_timeout() -> float | None:
     """Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
     if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
         return timeout
     return None
+
+
+class _DeploymentParams(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+def get_service_tier_timeout(
+    service_tier: object,
+    litellm_params: _DeploymentParams,
+    key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
+) -> float | None:
+    """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
+    if not isinstance(service_tier, str):
+        return None
+    timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
+    return None if timeouts is None else timeouts.get(service_tier)

litellm/llms/anthropic/chat/transformation.py

@@ -103,6 +103,7 @@ from ..common_utils import (
     process_anthropic_headers,
     requires_native_compaction_beta,
     strip_advisor_blocks_from_messages,
+    tools_marked_for_a_tool_config_point,
 )
 
 if TYPE_CHECKING:
@@ -329,6 +330,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
     def _resolved_provider(self) -> str:
         return self.custom_llm_provider or "anthropic"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AnthropicConfig
+
     @classmethod
     def get_config(cls, *, model: str | None = None):
         config: Final = super().get_config()
@@ -1656,7 +1661,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 else:
                     optional_params["thinking"] = value
                     AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
-                        model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider
+                        model=model,
+                        optional_params=optional_params,
+                        custom_llm_provider=self._resolved_provider,
+                        carry_block_binding=self._carries_thinking_block_binding,
                     )
             elif param == "reasoning_effort":
                 # Accept both string ("low") and dict ({"effort": "low",
@@ -2122,6 +2130,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         # Remove internal LiteLLM parameters that should not be sent to Anthropic API
         optional_params.pop("is_vertex_request", None)
         optional_params.pop("client_metadata", None)
+        request_params: Final = cast(  # cast-ok: optional_params is the untyped dict of the shared transform signature
+            dict[str, object], optional_params
+        )
+        marked_tools: Final = tools_marked_for_a_tool_config_point(
+            request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
+        )
+        if marked_tools is not None:
+            from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
+            request_litellm_params: Final = cast(  # cast-ok: litellm_params is the untyped dict of the shared signature
+                Mapping[str, object], litellm_params
+            )
+            AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1)
 
         # ``top_k`` is a provider-specific kwarg that bypasses
         # ``map_openai_params``; gate it here, the single boundary shared by
@@ -2141,6 +2162,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             "model": model,
             "messages": anthropic_messages,
             **optional_params,
+            **({} if marked_tools is None else {"tools": list(marked_tools)}),
         }
 
         self._apply_output_config(data=data, model=model, optional_params=optional_params)

litellm/llms/anthropic/common_utils.py

@@ -46,6 +46,7 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_OAUTH_BETA_HEADER,
     ANTHROPIC_OAUTH_TOKEN_PREFIX,
     ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
+    ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
     AllAnthropicToolsValues,
     AnthropicMcpServerTool,
     AnthropicMessagesToolChoice,
@@ -106,8 +107,8 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
 _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
 _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
-_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
-_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
+_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
+OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
 
 
 _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@@ -149,22 +150,61 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
     )
 
 
+def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
+    point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ())
+    tool_config_point: Final = next(
+        (fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None
+    )
+    if tool_config_point is None:
+        return None
+    control: Final = tool_config_point.get("control")
+    return {"type": "ephemeral"} if control is None else validated_object_mapping(control)
+
+
+def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None:
+    """Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches.
+
+    Anthropic caches tools as one prefix ending at the marked tool, so the last one carries
+    the mark, skipping tool-search tools whose transform drops cache markers. A client that
+    already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked.
+    """
+    control: Final = _tool_config_point_control(points)
+    tool_list: Final = validated_object_list(tools) or ()
+    tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list)
+    if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields):
+        return None
+    cacheable: Final = tuple(
+        (position, fields)
+        for position, fields in enumerate(tool_fields)
+        if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
+    )
+    if not cacheable:
+        return None
+    position, fields = cacheable[-1]
+    return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :])
+
+
 def is_claude_code_user_agent(user_agent: str) -> bool:
     """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/<version>` and its own
     fetches, such as gateway model discovery, as `claude-code/<version>`"""
     return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES)
 
 
-def _validated_claude_code_mapping(value: object) -> dict[object, object] | None:
+def validated_object_mapping(value: object) -> dict[object, object] | None:
     try:
-        return _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER.validate_python(value)
+        return _OBJECT_MAPPING_ADAPTER.validate_python(value)
     except ValidationError:
         return None
 
 
-def _validated_claude_code_list(value: object) -> list[object] | None:
+def thinking_block_binding(thinking: Mapping[object, object] | None) -> dict[str, object]:
+    block_binding: Final = thinking.get("block_binding") if thinking is not None else None
+    return {} if block_binding is None else {"block_binding": block_binding}
+
+
+def validated_object_list(value: object) -> Sequence[object] | None:
     try:
-        return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value)
+        return OBJECT_LIST_ADAPTER.validate_python(value)
     except ValidationError:
         return None
 
@@ -191,10 +231,10 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None
 def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None:
     if isinstance(system, str):
         return (system,)
-    blocks: Final = _validated_claude_code_list(system)
+    blocks: Final = validated_object_list(system)
     if blocks is None:
         return None
-    block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks)
+    block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks)
     if any(block is None for block in block_mappings):
         return None
     text_values: Final = tuple(
@@ -227,7 +267,7 @@ def is_claude_code_one_shot_subagent_request(
     tools: object,
     user_agent: str | None,
 ) -> bool:
-    only_message: Final = _validated_claude_code_mapping(messages[0]) if len(messages) == 1 else None
+    only_message: Final = validated_object_mapping(messages[0]) if len(messages) == 1 else None
     return (
         user_agent is not None
         and is_claude_code_user_agent(user_agent)
@@ -914,6 +954,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         model: str,
         optional_params: MutableMapping[str, object],  # mutable-ok: in-place out-param like the sibling helpers
         custom_llm_provider: str,
+        carry_block_binding: bool = False,
     ) -> None:
         """Translate legacy ``thinking.type=enabled`` to adaptive for the
         adaptive-thinking models that reject it (4.7+ and the 5 families).
@@ -926,21 +967,21 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return
         if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
             return
-        thinking: Final = optional_params.get("thinking")
-        if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
+        thinking: Final = validated_object_mapping(optional_params.get("thinking"))
+        if thinking is None or thinking.get("type") != "enabled":
             return
 
+        budget_tokens: Final = thinking.get("budget_tokens") or 0
         effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
             model=model,
-            budget_tokens=int(thinking.get("budget_tokens") or 0),
+            budget_tokens=int(budget_tokens) if isinstance(budget_tokens, (int, float, str)) else 0,
             custom_llm_provider=custom_llm_provider,
         )
         existing_output_config: Final = optional_params.get("output_config")
         display: Final = thinking.get("display")
-        if display in ("summarized", "omitted"):
-            optional_params["thinking"] = {"type": "adaptive", "display": display}
-        else:
-            optional_params["thinking"] = {"type": "adaptive"}
+        supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
+        carried_block_binding: Final = thinking_block_binding(thinking) if carry_block_binding else {}
+        optional_params["thinking"] = {**carried_block_binding, "type": "adaptive", **supported_display}
         optional_params["output_config"] = {
             "effort": effort,
             **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
@@ -1608,6 +1649,20 @@ def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
     return "must contain thinking" in lower
 
 
+def is_anthropic_thinking_binding_mismatch_error(error_text: str) -> bool:
+    """messages.N.content.M: Invalid `signature` in `thinking` block. The block is bound to a different conversation."""
+    return "bound to a different conversation" in error_text.lower()
+
+
+def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object) -> bool:
+    """``thinking.block_binding.prefix_mismatch_behavior: error`` is the caller asking
+    for the 400, so a retry that strips thinking must not override it."""
+    body: Final = validated_object_mapping(request_body)
+    thinking: Final = validated_object_mapping(body.get("thinking")) if body is not None else None
+    binding: Final = validated_object_mapping(thinking.get("block_binding")) if thinking is not None else None
+    return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
+
+
 def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
     """
     Return a new message list with thinking / redacted_thinking content blocks removed
@@ -1678,7 +1733,7 @@ def strip_empty_content_blocks_from_anthropic_messages(
 ) -> list[Any]:
     """
     Return a new message list with empty or whitespace-only ``{"type": "text"}``
-    and ``{"type": "thinking"}`` content blocks removed.
+    content blocks and empty *unsigned* ``{"type": "thinking"}`` blocks removed.
 
     Anthropic's API rejects requests containing such blocks with
     ``"messages: text content blocks must be non-empty"`` and
@@ -1694,8 +1749,13 @@ def strip_empty_content_blocks_from_anthropic_messages(
     on the unified ``/v1/messages`` path.  ``/v1/chat/completions`` already
     handles this in ``anthropic_messages_pt``; this helper provides the
     equivalent guarantee for the native Anthropic Messages path.
-    ``redacted_thinking`` blocks are never touched: they carry opaque
-    ``data`` instead of thinking text.
+
+    A thinking block with empty text but a non-empty ``signature`` is kept:
+    it is the default response shape of Claude Opus 4.8 / Fable 5 / Fable 5.1
+    (``thinking.display`` omitted), the signature is the model's preserved
+    reasoning, and Anthropic accepts it on input.  ``redacted_thinking``
+    blocks are never touched: they carry opaque ``data`` instead of thinking
+    text.
 
     Messages whose content is a list and becomes empty after stripping are
     omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
@@ -1708,7 +1768,7 @@ def strip_empty_content_blocks_from_anthropic_messages(
             out.append(m)
             continue
         content = m["content"]
-        filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)]
+        filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_unsigned_thinking_block(b)]
         if len(filtered) == len(content):
             out.append(m)
         elif filtered:
@@ -1726,11 +1786,11 @@ def _is_empty_text_block(block: object) -> bool:
 def is_empty_thinking_block(block: object) -> bool:
     """
     True for a ``{"type": "thinking"}`` content block whose thinking text is
-    missing, not a string, or empty/whitespace-only after ``.strip()``.
-    Anthropic rejects such blocks with ``"each thinking block must contain
-    thinking"`` (whitespace-only included, verified live), regardless of any
-    signature they carry.  ``redacted_thinking`` blocks are a different type
-    and always return False.
+    missing, not a string, or empty/whitespace-only after ``.strip()``,
+    whatever its signature.  This is a shape check only; whether such a block
+    may be sent to Anthropic depends on the signature, see
+    :func:`is_empty_unsigned_thinking_block`.  ``redacted_thinking`` blocks are
+    a different type and always return False.
     """
     if not isinstance(block, dict) or block.get("type") != "thinking":
         return False
@@ -1742,15 +1802,16 @@ def is_empty_unsigned_thinking_block(block: object) -> bool:
     """
     True for an empty ``{"type": "thinking"}`` block carrying no signature.
 
-    The emit-side predicate: response paths drop a thinking block only when it
-    holds nothing the client could need.  A signature-only block is a real
-    provider response (Bedrock Converse under adaptive thinking emits a
-    reasoning block with empty text and only a signature) and the client needs
-    the signature to replay reasoning across tool-use turns, so it must be
-    emitted.  Request paths keep using :func:`is_empty_thinking_block`:
-    Anthropic rejects empty thinking blocks in request history regardless of
-    signature, and the inbound strip self-heals a replayed signature-only
-    block.
+    Used on both the emit and the request side: a thinking block is dropped
+    only when it holds nothing the provider or the client could use.  A
+    signature-only block is a real provider response (Claude Opus 4.8 /
+    Fable 5 / Fable 5.1 return it by default, and Bedrock Converse under
+    adaptive thinking emits a reasoning block with empty text and only a
+    signature); the client needs the signature to replay reasoning across
+    tool-use turns, and Anthropic accepts the replayed block and binds it to
+    its prefix.  Only an empty block with no signature (a turn a
+    non-Anthropic reasoning model served through the bridge) is rejected
+    with ``"each thinking block must contain thinking"`` and must go.
     """
     if not isinstance(block, dict) or not is_empty_thinking_block(block):
         return False
@@ -1884,14 +1945,14 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None:
 
 
 def tool_call_is_rebuilt_as_server_tool_use(tool_call_id: object, provider_specific_fields: object) -> bool:
-    fields: Final = _validated_claude_code_mapping(provider_specific_fields)
+    fields: Final = validated_object_mapping(provider_specific_fields)
     if not isinstance(tool_call_id, str) or fields is None:
         return False
     return (
         find_anthropic_server_tool_result(
             tool_call_id,
-            _validated_claude_code_list(fields.get("web_search_results")),
-            _validated_claude_code_list(fields.get("tool_results")),
+            validated_object_list(fields.get("web_search_results")),
+            validated_object_list(fields.get("tool_results")),
         )
         is not None
     )