Merge branch 'main' into litellm_redis_transport_base_48956

.github/workflows/cost-map-guard.yml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@@ -21,16 +22,19 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head, its merge base, and the base branch tip
+      - name: Check out the base branch tip and fetch the pull request head and its merge base
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_REF: ${{ github.event.pull_request.base.ref }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
           merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
+          git checkout -q --detach "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
           echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv

.github/workflows/lens-install-smoke.yml

@@ -1,152 +0,0 @@
-name: Lens installation smoke
-
-on:
-  workflow_dispatch:
-    inputs:
-      lens_image:
-        description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      lens_upgrade_image:
-        description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      gateway_baseline_ref:
-        description: Previously qualified gateway source commit, full SHA
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: lens-install-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-images:
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-          - component: backend
-            dockerfile: backend/Dockerfile
-          - component: ui
-            dockerfile: ui/Dockerfile
-          - component: migrations
-            dockerfile: migrations/Dockerfile
-          - component: monolith
-            dockerfile: Dockerfile
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-            baseline: true
-          - component: backend
-            dockerfile: backend/Dockerfile
-            baseline: true
-          - component: monolith
-            dockerfile: Dockerfile
-            baseline: true
-    env:
-      COMPONENT: ${{ matrix.component }}
-      DOCKERFILE: ${{ matrix.dockerfile }}
-      SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
-      IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
-      CURRENT_REF: ${{ github.sha }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ env.SOURCE_REF }}
-      - name: Verify the selected source
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          test "$(git rev-parse HEAD)" = "$SOURCE_REF"
-          if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
-            test "$SOURCE_REF" != "$CURRENT_REF"
-          fi
-      - name: Build the matching release image
-        run: |
-          docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-            -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
-      - name: Save the matching release image
-        run: |
-          docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
-            | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
-          path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
-          compression-level: 0
-          retention-days: 3
-          if-no-files-found: error
-          overwrite: true
-
-  helm-install:
-    needs: build-images
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-install-*-${{ github.sha }}
-          merge-multiple: true
-          path: ${{ runner.temp }}/lens-install-images
-      - name: Load the matching release images
-        run: |
-          for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
-            gzip -dc "$archive" | docker load
-            rm "$archive"
-          done
-      - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
-        with:
-          cosign-release: v2.4.3
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ghcr.io
-          username: ${{ vars.LENS_READ_USER || github.actor }}
-          password: ${{ secrets.LENS_READ_TOKEN }}
-      - name: Verify independent Lens images before loading them
-        env:
-          LENS_IMAGE: ${{ inputs.lens_image }}
-          LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
-          LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
-          LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
-          printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
-          echo "$LENS_PUBLIC_KEY_SHA256  $RUNNER_TEMP/lens.pub" | sha256sum --check
-          for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
-            [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
-            cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
-            cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
-            docker pull "$image"
-          done
-          docker tag "$LENS_IMAGE" lens-ci-worker:baseline
-          docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
-      - name: Install pinned Kubernetes test tools
-        run: |
-          curl --fail --location --output "$RUNNER_TEMP/kind" \
-            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
-          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kind"
-          curl --fail --location --output "$RUNNER_TEMP/kubectl" \
-            https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
-          echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea  $RUNNER_TEMP/kubectl" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kubectl"
-          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
-            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
-          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
-          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
-          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
-      - name: Install, ingest, upgrade, and restart both charts
-        run: bash tests/e2e/migrations/lens_helm_smoke.sh

.github/workflows/spend-logs-guard.yml

@@ -19,17 +19,17 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          ref: ${{ github.event.pull_request.base.sha }}
+          ref: ${{ github.event.pull_request.base.ref }}
           persist-credentials: false
       - name: Fetch the pull request head, its merge base and its reviews
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

.github/workflows/test-linting.yml

@@ -48,11 +48,13 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$MERGE_BASE"
           retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
           echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@@ -264,6 +266,7 @@ jobs:
     name: ui
     permissions:
       contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     timeout-minutes: 8
     defaults:
@@ -281,16 +284,12 @@ jobs:
         id: changed
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          # base.sha is the base branch tip from when the PR was opened, while
-          # actions/checkout leaves HEAD on a merge of the PR into the *current*
-          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
-          # landed since, so a PR that touches no UI file still gets linted
-          # against hundreds of other people's files. Diff the PR head against its
-          # own merge base instead, which is exactly what this PR changed.
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"

.github/workflows/test-unit.yml

@@ -245,6 +245,7 @@ jobs:
               tests/unit/llms/chatgpt
               tests/unit/llms/clarifai
               tests/unit/llms/claude_code
+              tests/unit/llms/clf_ai_gateway
               tests/unit/llms/cloudflare
               tests/unit/llms/codestral
               tests/unit/llms/codex
@@ -1150,18 +1151,20 @@ jobs:
         env:
           CI: "true"
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
 
-          if [ -z "$BASE_SHA" ]; then
+          if [ -z "$PR_NUMBER" ]; then
             echo "Push to $GITHUB_REF_NAME: running the full suite"
             full_suite
             exit 0
           fi
 
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           changed_files=()

README.md

@@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

helm/litellm-helm/Chart.lock

@@ -5,8 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
-generated: '2026-10-09T04:18:10.643228000Z'
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: '2026-10-30T02:13:11.505726000Z'

helm/litellm-helm/Chart.yaml

@@ -39,6 +39,3 @@ dependencies:
     version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm-helm/README.md

@@ -256,9 +256,7 @@ itself.
 
 ## Connect Lens
 
-`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 ## Shutdown accounting
 

helm/litellm-helm/templates/_helpers.tpl

@@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
   value: {{ .Values.collector.drainTimeoutSeconds | quote }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -402,95 +398,28 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.labels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
-{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm-helm/templates/ingress.yaml

@@ -1,8 +1,6 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "litellm.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- $lensMode := include "litellm.lens.mode" . -}}
-{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@@ -46,18 +44,18 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
+          {{- if and $.Values.lens.enabled $.Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
-                name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
+                name: {{ $.Values.lens.service.name }}
                 port:
-                  number: {{ $.Values.lensWorker.service.port }}
+                  number: {{ $.Values.lens.service.port }}
               {{- else }}
-              serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
-              servicePort: {{ $.Values.lensWorker.service.port }}
+              serviceName: {{ $.Values.lens.service.name }}
+              servicePort: {{ $.Values.lens.service.port }}
               {{- end }}
           {{- end }}
           {{- range .paths }}

helm/litellm-helm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm-helm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm-helm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm-helm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm-helm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm-helm/tests/lens_connection_tests.yaml

@@ -0,0 +1,128 @@
+suite: lens connection
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - ingress.yaml
+tests:
+  - it: connects the proxy to an external Lens
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service ahead of the host paths
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[0].path
+          value: /lens-ingest
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,84 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - deployment.yaml
-  - configmap-litellm.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest

helm/litellm-helm/tests/lens_modes_tests.yaml

@@ -1,187 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- ingress.yaml
-- configmap-litellm.yaml
-- deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm-helm/tests/lens_modes_validation_tests.yaml

@@ -1,52 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm-helm/tests/lens_saved_secrets_tests.yaml

@@ -1,59 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

helm/litellm-helm/tests/lens_service_tests.yaml

@@ -1,167 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects deployment.yaml to the shared Lens service
-  template: deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.hosts:
-    - host: gateway.example
-      paths:
-      - path: /
-        pathType: Prefix
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in deployment.yaml
-  template: deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference

helm/litellm-helm/tests/lens_setup_tests.yaml

@@ -1,162 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60

helm/litellm-helm/values.yaml

@@ -668,70 +668,15 @@ serviceMonitor:
     # - test-namespace
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

helm/litellm/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
-generated: '2026-10-09T04:18:10.642706000Z'

helm/litellm/Chart.yaml

@@ -6,8 +6,3 @@ version: 0.1.0
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
-
-dependencies:
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm/README.md

@@ -1,10 +1,8 @@
 # LiteLLM componentized chart
 
-This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
+This chart deploys separate gateway, backend and UI services
 
-Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
 

helm/litellm/templates/_helpers.tpl

@@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -635,93 +631,27 @@ shutdown drain window.
 {{- end -}}
 
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -166,14 +166,14 @@ spec:
     - host: {{ $componentHosts.gateway | quote }}
       http:
         paths:
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           - path: /
             pathType: Prefix
@@ -249,15 +249,15 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           {{- /*
             --- Operator-supplied extra paths (ingress.extraPaths) ---

helm/litellm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm/tests/component_hosts_tests.yaml

@@ -15,7 +15,6 @@ tests:
       ingress.componentHosts.ui: ui.example.com
       ingress.componentHosts.backend: backend.example.com
       ingress.componentHosts.gateway: gateway.example.com
-      lensWorker.mode: disabled
       ingress.tls:
         - hosts:
             - ui.example.com
@@ -83,12 +82,12 @@ tests:
       ui.backendUrl: https://backend.example.com
       ui.publicUrl: https://ui.example.com
       backend.gatewayUrl: https://gateway.example.com
-      lensWorker.mode: external
-      lensWorker.externalServiceName: existing-lens
-      lensWorker.externalUrl: https://lens.example.com
-      lensWorker.publicUrl: https://lens.example.com
-      lensWorker.serviceTokenSecret.name: existing-lens-signing
-      lensWorker.gateway.secretName: existing-lens-signing
+      lens.enabled: true
+      lens.service.name: existing-lens
+      lens.url: http://existing-lens:4318
+      lens.publicUrl: https://gateway.example.com/lens-ingest
+      lens.serviceTokenSecret.name: existing-lens-signing
+      lens.gatewaySecret.name: existing-lens-signing
     asserts:
       - equal:
           path: spec.rules[2].http.paths[0].path

helm/litellm/tests/lens_connection_tests.yaml

@@ -0,0 +1,166 @@
+suite: lens connection
+templates:
+  - gateway/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: connects the gateway to an external Lens
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: connects the backend to an external Lens
+    template: backend/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service on the single ingress host
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,86 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - gateway/deployment.yaml
-  - gateway/configmap.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.host: gateway.example
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_modes_tests.yaml

@@ -1,208 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-- ingress.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: keeps the ingestion endpoint on an independently deployed Lens service
-  template: ingress.yaml
-  set:
-    ingress.enabled: true
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: existing-lens
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm/tests/lens_modes_validation_tests.yaml

@@ -1,54 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm/tests/lens_saved_secrets_tests.yaml

@@ -1,61 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_service_tests.yaml

@@ -1,196 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- gateway/configmap.yaml
-- gateway/deployment.yaml
-- backend/deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects gateway/deployment.yaml to the shared Lens service
-  template: gateway/deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: connects backend/deployment.yaml to the shared Lens service
-  template: backend/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.host: gateway.example
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: gateway/deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in gateway/deployment.yaml
-  template: gateway/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: omits Lens connection settings when disabled in backend/deployment.yaml
-  template: backend/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: gateway/deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference
-values:
-- ./values/required.yaml

helm/litellm/tests/lens_setup_tests.yaml

@@ -1,164 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_worker_tests.yaml

@@ -1,79 +0,0 @@
-suite: Independent Lens release and credentials
-templates:
-  - lens/deployment.yaml
-values:
-  - ./values/required.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-  lensWorker.serviceTokenSecret.name: lens-service
-  lensWorker.clickhouseSecret.name: lens-storage
-tests:
-  - it: selects a Lens release independently of the gateway
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-      lensWorker.image.tag: 2.3.4
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:2.3.4
-  - it: retains the pinned Lens chart default when only the gateway changes
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:0.1.0-dev.0
-  - it: selects development publications explicitly
-    set:
-      lensWorker.image.repository: ghcr.io/berriai/lens-dev
-      lensWorker.image.tag: sha-0123456789abcdef
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
-  - it: preserves a verified registry mirror and digest
-    set:
-      lensWorker.image.repository: registry.example/lens
-      lensWorker.image.tag: replaced-label
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: rejects a malformed digest
-    set:
-      lensWorker.image.digest: sha256:invalid
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
-  - it: keeps the legacy enablement switch working
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments: {count: 0}
-  - it: keeps isolated local analysis with delegated gateway access
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content: {name: LENS_MODE, value: standalone}
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_GATEWAY_SECRET
-            valueFrom:
-              secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
-      - equal:
-          path: spec.template.spec.automountServiceAccountToken
-          value: false
-      - equal:
-          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
-          value: true
-      - equal:
-          path: spec.template.spec.volumes[0].emptyDir
-          value: {medium: Memory, sizeLimit: 1Gi}

helm/litellm/values.yaml

@@ -654,70 +654,15 @@ ui:
   topologySpreadConstraints: []
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -238,7 +238,8 @@ impl AnthropicBeta {
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
-            | Self::ThinkingBindingControls20260801 => &[],
+            | Self::ThinkingBindingControls20260801
+            | Self::ThinkingDisplayUpdates20260818 => &[],
             Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
@@ -275,7 +276,6 @@ impl AnthropicBeta {
                 BetaProvider::Databricks,
             ],
             Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
             Self::ToolSearchTool20251019 => &[
                 BetaProvider::Anthropic,
                 BetaProvider::AzureAi,

litellm/__init__.py

@@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
 baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
+clf_ai_gateway_models: set[str] = set()  # mutable-ok: filled at import like sibling sets
 perplexity_models: Set = set()
 watsonx_models: Set = set()
 gemini_models: Set = set()
@@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             bedrock_converse_models.add(key)
         elif value.get("litellm_provider") == "deepinfra":
             deepinfra_models.add(key)
+        elif value.get("litellm_provider") == "clf_ai_gateway":  # pyright: ignore[reportUnknownMemberType]  # raw cost map
+            clf_ai_gateway_models.add(key)  # pyright: ignore[reportUnknownArgumentType]  # raw cost map key
         elif value.get("litellm_provider") == "perplexity":
             perplexity_models.add(key)
         elif value.get("litellm_provider") == "watsonx":
@@ -1110,6 +1113,7 @@ model_list = list(
     | set(ollama_models)
     | bedrock_models
     | deepinfra_models
+    | clf_ai_gateway_models
     | perplexity_models
     | set(maritalk_models)
     | runwayml_models
@@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict:
         "ollama": ollama_models,
         "ollama_chat": ollama_models,
         "deepinfra": deepinfra_models,
+        "clf_ai_gateway": clf_ai_gateway_models,
         "perplexity": perplexity_models,
         "maritalk": maritalk_models,
         "watsonx": watsonx_models,
@@ -2111,6 +2116,9 @@ if TYPE_CHECKING:
         LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
     )
     from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
+    from .llms.clf_ai_gateway.chat.transformation import (
+        ClfAiGatewayConfig as _ClfAiGatewayConfig,
+    )
     from .llms.llamafile.chat.transformation import (
         LlamafileChatConfig as _LlamafileChatConfig,
     )
@@ -2140,6 +2148,7 @@ if TYPE_CHECKING:
     IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
     LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
     DeepInfraConfig: Type[_DeepInfraConfig]
+    ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
     LlamafileChatConfig: Type[_LlamafileChatConfig]
     LMStudioChatConfig: Type[_LMStudioChatConfig]
     LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]

litellm/_lazy_imports_registry.py

@@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
     "MistralEmbeddingConfig",
     "OpenAIImageVariationConfig",
     "DeepInfraConfig",
+    "ClfAiGatewayConfig",
     "DeepgramAudioTranscriptionConfig",
     "TopazImageVariationConfig",
     "OpenAITextCompletionConfig",
@@ -975,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "OpenAIImageVariationConfig",
     ),
     "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
+    "ClfAiGatewayConfig": (
+        ".llms.clf_ai_gateway.chat.transformation",
+        "ClfAiGatewayConfig",
+    ),
     "DeepgramAudioTranscriptionConfig": (
         ".llms.deepgram.audio_transcription.transformation",
         "DeepgramAudioTranscriptionConfig",

litellm/caching/caching.py

@@ -18,7 +18,7 @@ from contextlib import contextmanager
 from enum import Enum
 from typing import Any, Final, Literal, cast
 
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 
 import litellm
 from litellm._internal_context import current_service_target, service_target
@@ -43,6 +43,9 @@ from .s3_cache import S3Cache
 RESPONSE_CACHE_TARGET: Final = "llm_response"
 _ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
 _DECODED_LITERAL: Final = TypeAdapter(object)
+_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
+    list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
 
 
 @contextmanager
@@ -207,7 +210,7 @@ class Cache:
             if not redis_startup_nodes:
                 _env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
                 if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
-                    redis_startup_nodes = json.loads(_env_cluster_nodes)
+                    redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
 
             if redis_startup_nodes:
                 # Only pass GCP parameters if they are provided

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
         self._structured_output: Final = structured_output
         self._first_message_item_done: bool = False
+        self._reasoning_summary_streamed: bool = False
 
     def _handle_string_chunk(
         self, str_line: Union[str, "BaseModel"]
@@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         tool_call_index_map: dict[int, int] | None = None,  # mutable-ok: per-stream state, remapped in place
         replays_openai_history: bool = False,
         structured_output: bool = False,
+        reasoning_summary_streamed: bool = False,
     ) -> "ModelResponseStream":
         """
         Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             tool_call_index_map: Per-stream output_index -> sequential tool_call index map
             replays_openai_history: Record the completed output's item sequence for the next turn
             structured_output: Only the first message item became content
+            reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
 
         Returns:
             ModelResponseStream: OpenAI-formatted streaming chunk
@@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 )
             else:
                 raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
+        elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
+            return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
         elif event_type == "response.reasoning_summary_text.delta":
             content_part = parsed_chunk.get("delta", None)
             if content_part:
                 return ModelResponseStream(
                     choices=[
                         StreamingChoices(
-                            index=cast(int, parsed_chunk.get("summary_index")),
+                            index=0,
                             delta=Delta(reasoning_content=content_part),
                         )
                     ]
@@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 tool_call_index_map=self._tool_call_index_map,
                 replays_openai_history=self._replays_openai_history,
                 structured_output=self._structured_output,
+                reasoning_summary_streamed=self._reasoning_summary_streamed,
             )
         )
         if self._structured_output and _is_message_item_done(chunk):
             self._first_message_item_done = True
+        if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
+            self._reasoning_summary_streamed = True
         return self._with_served_service_tier(self._with_stream_scoped_id(translated))
 
     def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

litellm/constants.py

@@ -7,6 +7,7 @@ from typing import Final, Literal
 
 from litellm.litellm_core_utils.env_utils import (
     get_env_float,
+    get_env_float_in_range,
     get_env_float_or_none,
     get_env_int,
     get_env_int_in_range,
@@ -149,6 +150,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
     "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
 )  # default cooldown a deployment if 50% of requests fail in a given minute
 DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
+NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
+    "NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
+)
 DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
 DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
 DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
@@ -819,6 +823,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
     "ollama",
     "ollama_chat",
     "deepinfra",
+    "clf_ai_gateway",
     "perplexity",
     "mistral",
     "groq",
@@ -1018,6 +1023,7 @@ openai_compatible_endpoints: Final[list[str]] = [
     "api.perplexity.ai",
     "api.endpoints.anyscale.com/v1",
     "api.deepinfra.com/v1/openai",
+    "api.clfaigateway.dev/v1",
     "api.mistral.ai/v1",
     "codestral.mistral.ai/v1/chat/completions",
     "codestral.mistral.ai/v1/fim/completions",
@@ -1089,6 +1095,7 @@ openai_compatible_providers: Final[list[str]] = [
     "deepseek",
     "tencent",
     "deepinfra",
+    "clf_ai_gateway",
     "perplexity",
     "xinference",
     "xai",

litellm/cost_calculator.py

@@ -153,7 +153,9 @@ from litellm.utils import (
     ProviderConfigManager,
     TextCompletionResponse,
     TranscriptionResponse,
+    cached_cost_map_key,
     cached_get_model_info_helper,
+    mapped_model_fill_missing,
     model_info_with_pricing_overrides,
     runtime_model_cost_registrations,
     token_counter,
@@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
     """
     1. If custom pricing is true, return received model name
     2. If base_model is set (e.g. for azure models), return that
-    3. If completion response has model set return that
+    3. If completion response has model set return that, unless only the requested model's cost-map row has rates
     4. Check if model is passed in return that
     """
 
     return_model: str | None = None
     custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
 
-    completion_response_model: str | None = None
-    if completion_response is not None:
-        if isinstance(completion_response, BaseModel):
-            completion_response_model = getattr(completion_response, "model", None)
-        elif isinstance(completion_response, dict):
-            completion_response_model = completion_response.get("model", None)
+    completion_response_model: Final = _get_response_model(completion_response)
     hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
+    requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
     configured_model: Final = model or (
         completion_response_model if isinstance(completion_response_model, str) else None
@@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
         else:
             return_model = model
 
-    elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
-        return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
+    elif base_model is not None or bedrock_pricing_model is not None:
+        return_model = base_model if base_model is not None else bedrock_pricing_model
+
+    elif provider_response_model is not None:
+        return _served_or_priced_requested_model(
+            provider_response_model, requested_model, custom_llm_provider, priced_region
+        )
 
     elif completion_response_model is None and hidden_params is not None:
         if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
             return_model = hidden_params.get("model", model)
 
     if return_model is None and completion_response_model is not None:
-        return_model = completion_response_model
+        return _served_or_priced_requested_model(
+            completion_response_model, requested_model, custom_llm_provider, priced_region
+        )
 
     if return_model is None and model is not None:
         return_model = model
 
-    if (
-        return_model is not None
-        and custom_llm_provider is not None
-        and not _model_contains_known_llm_provider(return_model)
-    ):  # add provider prefix if not already present, to match model_cost
-        provider_prefix: Final = (
-            custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
-        )
-        return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
-
-    return return_model
+    if return_model is None:
+        return None
+    return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
 
 
 _select_model_name_for_cost_calc = select_model_name_for_cost_calc
 
 
+def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
+    if custom_llm_provider is None or _model_contains_known_llm_provider(model):
+        return model
+    provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
+    return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
+
+
+def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
+    rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
+    return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
+
+
+def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
+    key: Final = cached_cost_map_key(model, custom_llm_provider)
+    if key is None:
+        return False
+    entry: Final = _raw_cost_map_entry(key)
+    if entry is None:
+        return _rule_declares_rates(model, custom_llm_provider)
+    fill_missing: Final = mapped_model_fill_missing(key, entry)
+    return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
+
+
+def _served_or_priced_requested_model(
+    served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
+) -> str:
+    served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
+    if requested_model is None:
+        return served
+    requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
+    if served in (requested, requested.partition("/")[2]):
+        return served
+    pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
+    if _resolves_to_rates(served, pricing_provider):
+        return served
+    return requested if _resolves_to_rates(requested, pricing_provider) else served
+
+
 def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
     """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
     registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@@ -1509,6 +1544,12 @@ def _response_time_ms_for_cost(
     return litellm_logging_obj.get_response_ms()
 
 
+def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
+    breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
+    total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
+    return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
+
+
 def _adjusted_provider_billed_cost(
     reported_cost: float,
     custom_llm_provider: str | None,
@@ -1526,9 +1567,9 @@ def _adjusted_provider_billed_cost(
     )
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=None,
-        completion_tokens_cost_usd_dollar=None,
-        cost_for_built_in_tools_cost_usd_dollar=None,
+        prompt_tokens_cost_usd_dollar=0.0,
+        completion_tokens_cost_usd_dollar=adjusted_cost,
+        cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=adjusted_cost,
         original_cost=reported_cost,
         discount_percent=discount_percent,
@@ -2341,6 +2382,15 @@ def response_cost_calculator(
                     )
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None and not custom_pricing:
+                        if _breakdown_total(litellm_logging_obj) != provider_response_cost:
+                            _store_cost_breakdown_in_logging_obj(
+                                litellm_logging_obj=litellm_logging_obj,
+                                prompt_tokens_cost_usd_dollar=0.0,
+                                completion_tokens_cost_usd_dollar=provider_response_cost,
+                                cost_for_built_in_tools_cost_usd_dollar=0.0,
+                                total_cost_usd_dollar=provider_response_cost,
+                                original_cost=provider_response_cost,
+                            )
                         return provider_response_cost
 
             response_cost = completion_cost(
@@ -3516,7 +3566,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
     lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
     session that should bill nothing fell through to the public rates instead.
     """
-    resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
+    resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
     entries: Final = (
         litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
         litellm.model_cost.get(model_name),
@@ -3712,9 +3762,9 @@ def handle_realtime_transcription_cost_calculation(
         return 0.0
 
     model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
+    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
     override_info: Final = (
-        _get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
+        _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
 
     total_cost = 0.0
@@ -3745,13 +3795,6 @@ def _get_transcription_model_name_from_results(
     return None
 
 
-def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
-    try:
-        return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
-    except Exception:
-        return None
-
-
 _TranscriptionRateKey: TypeAlias = Literal[
     "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
 ]

litellm/images/utils.py

@@ -20,7 +20,7 @@ class ImageEditRequestUtils:
         image_edit_optional_params: ImageEditOptionalRequestParams,
         drop_params: bool | None = None,
         additional_drop_params: Collection[str] | None = None,
-    ) -> dict:
+    ) -> dict[str, object]:
         """
         Get optional parameters for the image edit API.
 

litellm/integrations/arize/arize_phoenix_client.py

@@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 import urllib.parse
 from typing import Final
 
+from pydantic import BaseModel, ConfigDict
+
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+class _PromptVersionResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
+
+    data: dict[str, object] | None = None
+
+
 def _sanitize_id(identifier: str) -> str:
     """Reject path traversal characters and URL-encode the identifier."""
     if any(c in identifier for c in ("/", "\\", "#", "?")):
@@ -71,8 +79,7 @@ class ArizePhoenixClient:
             response: Final = self.http_handler.get(url, headers=self.headers)
             response.raise_for_status()
 
-            data: Final = response.json()
-            return data.get("data")
+            return _PromptVersionResponse.model_validate(response.json()).data
 
         except Exception as e:
             # Check if it's an HTTP error

litellm/integrations/opentelemetry.py

@@ -72,7 +72,7 @@ if TYPE_CHECKING:
     )
     from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
-    Span = _Span | Any
+    Span = _Span
     Tracer = _Tracer
     Context = _Context
     SpanExporter = _SpanExporter
@@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             span = None
             # Only set attributes if the span is still recording (not closed)
             # Note: parent_span is guaranteed to be not None here
-            if hasattr(parent_span, "set_status"):
+            if parent_span is not None and hasattr(parent_span, "set_status"):
                 parent_span.set_status(Status(StatusCode.OK))
                 self.set_attributes(parent_span, kwargs, response_obj)
             # Raw-request as direct child of parent_span
@@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             # record error on parent span (keeps hierarchy shallow)
             # Only set attributes if the span is still recording (not closed)
             # Note: parent_otel_span is guaranteed to be not None here
-            if parent_otel_span.is_recording():
+            if parent_otel_span is not None and parent_otel_span.is_recording():
                 self.set_attributes(parent_otel_span, kwargs, response_obj)
                 parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
                 parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))

litellm/litellm_core_utils/core_helpers.py

@@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
     return getattr(chunk, "usage", None)
 
 
-USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
+PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
+
+
+UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
+
+
+def _numeric(value: object) -> float | None:
+    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
+
+
+def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
+    """Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
+    ``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
+    reading litellm's bill back as if the upstream had reported it."""
+    hidden: Final[object] = getattr(response, "_hidden_params", None)
+    if isinstance(hidden, dict):
+        hidden_params: Final = cast(dict[str, object], hidden)  # cast-ok: hidden params ride on the response as a dict
+        upstream: Final = (
+            hidden_params[UPSTREAM_USAGE_COST]
+            if UPSTREAM_USAGE_COST in hidden_params
+            else _numeric(getattr(usage, "cost", None))
+        )
+        hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
+    setattr(usage, "cost", bill)
 
 
 def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
-    if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
+    """The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
+    usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
+    event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
+    if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
         return None
-    cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
+    cost: Final = (
+        _numeric(hidden[UPSTREAM_USAGE_COST])
+        if hidden is not None and UPSTREAM_USAGE_COST in hidden
+        else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    )
     return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
 
 

litellm/litellm_core_utils/env_utils.py

@@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
     Values outside the range fall back to the default and warn, so a misconfigured knob can
     neither crash the caller nor silently change the meaning of what it computes.
     """
-    value: Final = get_env_int(env_var, default)
+    return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
+
+
+def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
+    """Float counterpart of `get_env_int_in_range`."""
+    return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
+
+
+def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
     if minimum <= value <= maximum:
         return value
     logging.getLogger("LiteLLM").warning(

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -292,6 +292,10 @@ def get_llm_provider(
                     elif endpoint == "api.deepinfra.com/v1/openai":
                         custom_llm_provider = "deepinfra"
                         dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
+                    elif endpoint == "api.clfaigateway.dev/v1":
+                        custom_llm_provider = "clf_ai_gateway"  # rebind-ok: endpoint inference, like every branch
+                        clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
+                        dynamic_api_key = api_key or clf_env_key
                     elif endpoint == "api.mistral.ai/v1":
                         custom_llm_provider = "mistral"
                         dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
@@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "clf_ai_gateway":
+        (
+            api_base,  # rebind-ok: resolved base/key are what this function returns, like every branch
+            dynamic_api_key,  # rebind-ok: see above
+        ) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "empower":
         api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
         dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")

litellm/litellm_core_utils/litellm_logging.py

@@ -202,6 +202,7 @@ from litellm.types.utils import (
     TextCompletionResponse,
     TranscriptionResponse,
     Usage,
+    batch_chat_usage,
 )
 from litellm.types.videos.main import VideoObject
 from litellm.utils import get_base_model_from_metadata, print_verbose
@@ -2091,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
             elif router_model_id is None and "model_id" in hidden_params:  # use model_id if not already set
                 router_model_id = hidden_params["model_id"]
 
+        if isinstance(priced_result, LiteLLMBatch) and (
+            not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
+        ):
+            return 0.0
+
         # Fallback: extract router_model_id from litellm_params when not available
         # from the result object. ResponsesAPIResponse objects (used by /v1/responses
         # streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
@@ -3346,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
         finally:
             self._restore_correlation_context()
 
+    def _is_batch_poll_without_logging(self, result: object) -> bool:
+        litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
+        return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
+
     def _success_handler_body(
         self,
         result: object = None,
@@ -3860,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## CALCULATE COST FOR BATCH JOBS
         if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
-            litellm_params = self.litellm_params or {}
-            litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
-            if (
-                litellm_metadata.get("batch_ignore_default_logging", False) is True
-            ):  # polling job will query these frequently, don't spam db logs
+            if self._is_batch_poll_without_logging(result):
                 return
 
             batch_cost: Final = kwargs.get("batch_cost", None)
@@ -7481,7 +7487,8 @@ def get_standard_logging_object_payload(
         # Extract usage as a plain dict, avoiding Pydantic round-trip
         raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
             response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
-            combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
+            combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
+            or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
         )
         usage_dict: Final = (
             {**raw_usage_dict, "output_image_count": len(init_response_obj.data)}

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -5878,38 +5878,36 @@ def function_call_prompt(
     ]
 
 
-def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
+def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
     """
     Decides if a user-defined custom prompt or default needs to be used
 
     Returns the prompt str that's passed to the model as a user message
     """
-    custom_prompt_details: dict | None = None
-    response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
-    if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
-        custom_prompt_details = litellm.custom_prompt_dict[
-            f"{model}/response_schema_prompt"
-        ]  # allow user to define custom response schema prompt by model
-    elif "response_schema_prompt" in litellm.custom_prompt_dict:
-        custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
-
-    if custom_prompt_details is not None:
-        return render_custom_prompt_template(
-            template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
-        )
-    else:
+    prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
+    model_prompt_key: Final = f"{model}/response_schema_prompt"
+    custom_prompt_details: Final = prompts.get(
+        model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
+    )
+    if custom_prompt_details is None:
         return default_response_schema_prompt(response_schema=response_schema)
+    return render_custom_prompt_template(
+        template=custom_prompt_details,
+        messages=[{"role": "user", "content": f"{response_schema}"}],
+        model=model,
+        llm_provider=llm_provider,
+    )
 
 
-def default_response_schema_prompt(response_schema: dict) -> str:
+def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
     """
     Used if provider/model doesn't support 'response_schema' param.
 
     This is the default prompt. Allow user to override this with a custom_prompt.
     """
     prompt_str: Final = f"""Use this JSON schema: 
-    ```json 
-    {response_schema}
+    ```json
+    {json.dumps(response_schema, default=str)}
     ```"""
     return prompt_str
 

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -3,7 +3,7 @@ import builtins
 import time
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, replace
-from itertools import chain, groupby
+from itertools import chain, groupby, takewhile
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
 
@@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import (
-    USAGE_COST_BILLED_PROVIDERS,
+    PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
     chunk_usage,
     provider_reported_usage_cost,
+    stamp_litellm_bill_on_usage,
 )
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
@@ -257,7 +258,7 @@ def apply_grounding_request_counts(
 
 
 class ChunkProcessor:
-    def __init__(self, chunks: list, messages: Sequence | None = None):
+    def __init__(self, chunks: list, messages: Sequence[object] | None = None):
         self.chunks = self._sort_chunks(chunks)
         self.messages = messages
         self.first_chunk = chunks[0]
@@ -390,6 +391,30 @@ class ChunkProcessor:
             case _:
                 return "assistant"
 
+    @staticmethod
+    def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object:
+        finish_reason = unreported
+        for chunk in chunks:
+            if "choices" in chunk and len(chunk["choices"]) > 0:
+                chunk_finish_reason = None
+                if hasattr(chunk["choices"][0], "finish_reason"):
+                    chunk_finish_reason = chunk["choices"][0].finish_reason
+                elif "finish_reason" in chunk["choices"][0]:
+                    chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
+                if chunk_finish_reason is not None:
+                    finish_reason = chunk_finish_reason
+        return finish_reason
+
+    @staticmethod
+    def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices:
+        return Choices(
+            index=index,
+            finish_reason=ChunkProcessor._finish_reason_of_chunks(
+                chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
+            ),
+            message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
+        )
+
     @staticmethod
     def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
         """
@@ -414,7 +439,11 @@ class ChunkProcessor:
             None,
         )
 
-    def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
+    def build_base_response(
+        self,
+        chunks: Sequence["_BaseChunk"],
+        choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
+    ) -> ModelResponse:
         chunk = self.first_chunk
         id: Final = ChunkProcessor._get_chunk_id(chunks)
         object: Final = chunk["object"]
@@ -426,16 +455,7 @@ class ChunkProcessor:
         service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks)
 
         role: Final = ChunkProcessor._get_role_from_chunks(chunks)
-        finish_reason = "stop"
-        for chunk in chunks:
-            if "choices" in chunk and len(chunk["choices"]) > 0:
-                chunk_finish_reason = None
-                if hasattr(chunk["choices"][0], "finish_reason"):
-                    chunk_finish_reason = chunk["choices"][0].finish_reason
-                elif "finish_reason" in chunk["choices"][0]:
-                    chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
-                if chunk_finish_reason is not None:
-                    finish_reason = chunk_finish_reason
+        finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
 
         # Initialize the response dictionary
         response = ModelResponse(
@@ -455,13 +475,17 @@ class ChunkProcessor:
                     if service_tier is not None
                     else MappingProxyType({})
                 ),
-                "choices": [
-                    {
-                        "index": 0,
-                        "message": {"role": role, "content": ""},
-                        "finish_reason": finish_reason,
-                    }
-                ],
+                "choices": (
+                    [ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
+                    if len(choice_streams) > 1
+                    else [
+                        {
+                            "index": 0,
+                            "message": {"role": role, "content": ""},
+                            "finish_reason": finish_reason,
+                        }
+                    ]
+                ),
                 "usage": {
                     "prompt_tokens": 0,  # Modify as needed
                     "completion_tokens": 0,  # Modify as needed
@@ -1357,9 +1381,6 @@ def stream_chunk_builder_text_completion(
     return TextCompletionResponse(**response)
 
 
-_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
-
-
 def _logged_provider(logging_obj: Optional["Logging"]) -> object:
     if logging_obj is None:
         return None
@@ -1367,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
     return provider
 
 
-def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
-    return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
-
-
 def _stream_builder_response_cost(
     response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
 ) -> float | None:
@@ -1381,9 +1398,6 @@ def _stream_builder_response_cost(
             for chunk in reversed(chunks)
         )
         return next((cost for cost in reported if cost is not None), None)
-    usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
-    if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
-        return float(usage_cost)
     if logging_obj is not None:
         return None
     provider_hint: Final = response._hidden_params.get(  # pyright: ignore[reportPrivateUsage]  # no public accessor
@@ -1430,13 +1444,15 @@ def _set_stream_builder_response_cost(
 def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
     if logging_obj is None:
         return
-    if isinstance(getattr(usage, "cost", None), (int, float)):
+    if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
+        getattr(usage, "cost", None), (int, float)
+    ):
         return
     computed_cost: Final = logging_obj.response_cost_calculator(result=response)
-    if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
+    if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
         return
-    setattr(usage, "cost", computed_cost)
-    _set_hidden_response_cost(response, float(computed_cost))
+    if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
+        stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
 
 
 def _field_of(value: object, name: str) -> object:
@@ -1520,11 +1536,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
     return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
 
 
+def _stream_choice_index(choice: object) -> int:
+    index: Final = _field_of(choice, "index")
+    return index if isinstance(index, int) else 0
+
+
+def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool:
+    return any(
+        "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"]
+        for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks)
+    )
+
+
+def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]:
+    indices: Final = sorted(
+        frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks)))
+    )
+    if len(indices) <= 1:
+        return ((0, chunks),)
+    return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices)
+
+
+def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]:
+    return tuple(
+        _BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index])
+        for chunk in chunks
+    )
+
+
+def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None:
+    choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices"))
+    parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk)))
+    return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None
+
+
+def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]:
+    tool_call_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if any(
+            "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
+            for choice in chunk.get("choices") or ()
+        )
+    ]
+
+    if tool_call_chunks:
+        tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
+        yield "content", None
+        yield "tool_calls", tool_calls_list
+
+    function_call_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "function_call" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["function_call"] is not None
+    ]
+
+    if function_call_chunks:
+        yield "content", None
+        yield "function_call", processor.get_combined_function_call_content(function_call_chunks)
+
+    content_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "content" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["content"] is not None
+    ]
+
+    if content_chunks:
+        yield "content", processor.get_combined_content(content_chunks)
+
+    thinking_blocks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "thinking_blocks" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
+    ]
+
+    if thinking_blocks:
+        yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks)
+
+    reasoning_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "reasoning_content" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["reasoning_content"] is not None
+    ]
+
+    if reasoning_chunks:
+        yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
+
+    typed_chunks: Final = cast(Sequence[object], chunks)  # cast-ok: chunks are untyped upstream
+    reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
+    if reasoning_items:
+        yield "reasoning_items", list(reasoning_items)
+
+    annotation_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "annotations" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["annotations"] is not None
+    ]
+
+    message_annotations, provider_annotations = (
+        partition_annotations(
+            list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks))
+        )
+        if annotation_chunks
+        else (None, ())
+    )
+    if message_annotations is not None:
+        yield "annotations", message_annotations
+
+    audio_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "audio" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["audio"] is not None
+    ]
+
+    if audio_chunks:
+        yield "audio", processor.get_combined_audio_content(audio_chunks)
+
+    # Handle image chunks from models like gemini-2.5-flash-image
+    # See: https://github.com/BerriAI/litellm/issues/19478
+    image_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "images" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["images"] is not None
+    ]
+
+    if image_chunks:
+        # Images come complete in a single chunk, collect all images from all chunks
+        all_images: Final = []
+        for chunk in image_chunks:
+            all_images.extend(chunk["choices"][0]["delta"]["images"])
+        yield "images", all_images
+
+    # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
+    # See: https://github.com/BerriAI/litellm/issues/17737
+    provider_specific_chunks: Final = [
+        chunk
+        for chunk in chunks
+        if chunk.get("choices")
+        and "provider_specific_fields" in chunk["choices"][0]["delta"]
+        and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
+    ]
+
+    provider_field_dicts: Final = tuple(
+        fields
+        for chunk in provider_specific_chunks
+        for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
+        if isinstance(fields, dict)
+    )
+    streamed_citations: Final = tuple(
+        fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
+    )
+    citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
+    provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {}
+    combined_provider_fields: Final = {
+        key: value
+        for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts)
+        for key, value in fields.items()
+        if key != "citation"
+    }
+
+    if combined_provider_fields:
+        yield "provider_specific_fields", combined_provider_fields
+
+
 def stream_chunk_builder(
     chunks: list,
     messages: Sequence | None = None,
-    start_time=None,
-    end_time=None,
+    start_time: object = None,
+    end_time: object = None,
     logging_obj: Optional["Logging"] = None,
     count_prompt_tokens: Callable[[], int] | None = None,
 ) -> ModelResponse | TextCompletionResponse | None:
@@ -1561,26 +1754,15 @@ def stream_chunk_builder(
         usage_model: Final = usage_tokenizer_model(
             model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
         )
-        # Initialize the response dictionary
-        response: Final = processor.build_base_response(chunks)
+        choice_streams: Final = _choice_streams(chunks)
+        response: Final = processor.build_base_response(chunks, choice_streams)
+        rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
 
-        # Fast path for the common text-only streaming case:
-        # avoid repeated multi-pass list scans over chunks.
-        simple_content_parts: Final[list[str]] = []
-        is_simple_text_stream = True
-        for chunk in chunks:
-            if not chunk.get("choices"):
-                continue
-
-            if (part := _simple_text_part(chunk["choices"])) is None:
-                is_simple_text_stream = False
-                break
-            if part:
-                simple_content_parts.append(part)
-
-        if is_simple_text_stream:
-            if simple_content_parts:
-                response["choices"][0]["message"]["content"] = "".join(simple_content_parts)
+        simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams)
+        if None not in simple_texts:
+            for (choice, _), simple_text in zip(rebuilt_choices, simple_texts):
+                if simple_text:
+                    choice.message.content = simple_text
             completion_output = get_content_from_model_response(response)
             usage = processor.calculate_usage(
                 chunks=chunks,
@@ -1619,165 +1801,9 @@ def stream_chunk_builder(
             processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
             return response
 
-        tool_call_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if any(
-                "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
-                for choice in chunk.get("choices") or ()
-            )
-        ]
-
-        if len(tool_call_chunks) > 0:
-            tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
-            _choice = response.choices[0]
-            _choice.message.content = None
-            _choice.message.tool_calls = tool_calls_list
-
-        function_call_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "function_call" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["function_call"] is not None
-        ]
-
-        if len(function_call_chunks) > 0:
-            _choice = response.choices[0]
-            _choice.message.content = None
-            _choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks)
-
-        content_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "content" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["content"] is not None
-        ]
-
-        if len(content_chunks) > 0:
-            response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks)
-
-        thinking_blocks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "thinking_blocks" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
-        ]
-
-        if len(thinking_blocks) > 0:
-            response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content(
-                thinking_blocks
-            )
-
-        reasoning_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "reasoning_content" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["reasoning_content"] is not None
-        ]
-
-        if len(reasoning_chunks) > 0:
-            response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content(
-                reasoning_chunks
-            )
-
-        typed_chunks: Final = cast(Sequence[object], chunks)  # cast-ok: chunks are untyped upstream
-        reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
-        if reasoning_items:
-            response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
-
-        annotation_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "annotations" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["annotations"] is not None
-        ]
-
-        if len(annotation_chunks) > 0:
-            # Merge annotations from ALL chunks — providers may spread
-            # them across multiple streaming chunks or send them only in
-            # the final chunk.
-            all_annotations: Final[list[object]] = []
-            for ac in annotation_chunks:
-                all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
-            message_annotations, provider_annotations = partition_annotations(all_annotations)
-            if message_annotations is not None:
-                response["choices"][0]["message"]["annotations"] = message_annotations
-            if provider_annotations:
-                response.choices[0].message.provider_specific_fields = {
-                    **(response.choices[0].message.provider_specific_fields or {}),
-                    "annotations": list(provider_annotations),
-                }
-
-        audio_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "audio" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["audio"] is not None
-        ]
-
-        if len(audio_chunks) > 0:
-            _choice = response.choices[0]
-            _choice.message.audio = processor.get_combined_audio_content(audio_chunks)
-
-        # Handle image chunks from models like gemini-2.5-flash-image
-        # See: https://github.com/BerriAI/litellm/issues/19478
-        image_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "images" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["images"] is not None
-        ]
-
-        if len(image_chunks) > 0:
-            # Images come complete in a single chunk, collect all images from all chunks
-            all_images: Final = []
-            for chunk in image_chunks:
-                all_images.extend(chunk["choices"][0]["delta"]["images"])
-            response["choices"][0]["message"]["images"] = all_images
-
-        # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
-        # See: https://github.com/BerriAI/litellm/issues/17737
-        provider_specific_chunks: Final = [
-            chunk
-            for chunk in chunks
-            if chunk.get("choices")
-            and "provider_specific_fields" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
-        ]
-
-        if len(provider_specific_chunks) > 0:
-            provider_field_dicts: Final = tuple(
-                fields
-                for chunk in provider_specific_chunks
-                for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
-                if isinstance(fields, dict)
-            )
-            streamed_citations: Final = tuple(
-                fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
-            )
-            citation_fields: Final = (
-                {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
-            )
-            combined_provider_fields: Final = {
-                key: value
-                for fields in (citation_fields, *provider_field_dicts)
-                for key, value in fields.items()
-                if key != "citation"
-            }
-
-            if combined_provider_fields:
-                _choice = response.choices[0]
-                _choice.message.provider_specific_fields = {
-                    **(_choice.message.provider_specific_fields or {}),
-                    **combined_provider_fields,
-                }
+        for choice, choice_chunks in rebuilt_choices:
+            for field, value in _assembled_message_fields(processor, choice_chunks):
+                choice.message[field] = value
 
         completion_output = get_content_from_model_response(response)
 

litellm/llms/a2a/chat/guardrail_translation/handler.py

@@ -14,6 +14,7 @@ import json
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Final, Optional
 
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
@@ -28,6 +29,8 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 class _A2ATextPart(TypedDict, total=False):
     """The subset of an A2A message part this handler reads text from."""
@@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation):
                 obj = item
             elif isinstance(item, str):
                 try:
-                    obj = json.loads(item.strip())
-                except (json.JSONDecodeError, TypeError):
+                    obj = _JSON_OBJECT.validate_python(json.loads(item.strip()))
+                except (json.JSONDecodeError, TypeError, ValidationError):
                     continue
             else:
                 continue

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -18,9 +18,9 @@ from copy import deepcopy
 from dataclasses import dataclass
 from itertools import chain, repeat
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
 from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm._logging import verbose_proxy_logger
@@ -349,14 +349,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...
     )
 
 
-class _AnthropicSSEDelta(TypedDict, total=False):
-    type: ReadOnly[str]
+@with_config(ConfigDict(extra="ignore", strict=True))
+class _SSETextDelta(TypedDict, total=False):
+    type: ReadOnly[object]
     text: ReadOnly[str]
-    stop_reason: ReadOnly[str | None]
 
 
-class _AnthropicSSEEvent(TypedDict, total=False):
-    delta: ReadOnly[_AnthropicSSEDelta]
+@with_config(ConfigDict(extra="ignore", strict=True))
+class _SSETextDeltaEvent(TypedDict, total=False):
+    delta: ReadOnly[_SSETextDelta]
+
+
+@with_config(ConfigDict(extra="ignore", strict=True))
+class _SSEStopDelta(TypedDict, total=False):
+    stop_reason: ReadOnly[object]
+
+
+@with_config(ConfigDict(extra="ignore", strict=True))
+class _SSEStopDeltaEvent(TypedDict, total=False):
+    delta: ReadOnly[_SSEStopDelta]
+
+
+_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent)
+_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent)
+_SSEEventT = TypeVar("_SSEEventT")
+
+
+def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None:
+    try:
+        return adapter.validate_python(json.loads(data_line))
+    except json.JSONDecodeError:
+        verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type)
+    except ValidationError:
+        verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type)
+    return None
 
 
 class AnthropicMessagesHandler(BaseTranslation):
@@ -1806,13 +1832,10 @@ class AnthropicMessagesHandler(BaseTranslation):
 
                 # Only process content_block_delta events
                 if event_type == "content_block_delta" and data_line:
-                    try:
-                        data: _AnthropicSSEEvent = json.loads(data_line)
-                        delta: _AnthropicSSEDelta = data.get("delta", {})
-                        if delta.get("type") == "text_delta":
-                            text += delta.get("text", "")
-                    except json.JSONDecodeError:
-                        verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
+                    data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line)
+                    delta: _SSETextDelta = {} if data is None else data.get("delta", {})
+                    if delta.get("type") == "text_delta":
+                        text += delta.get("text", "")
 
         except Exception as e:
             verbose_proxy_logger.error("Error extracting text from SSE: %s", e)
@@ -1869,14 +1892,11 @@ class AnthropicMessagesHandler(BaseTranslation):
 
                         # Check for message_delta event with stop_reason
                         if event_type == "message_delta" and data_line:
-                            try:
-                                data: _AnthropicSSEEvent = json.loads(data_line)
-                                delta: _AnthropicSSEDelta = data.get("delta", {})
-                                stop_reason: str | None = delta.get("stop_reason")
-                                if stop_reason is not None:
-                                    return True
-                            except json.JSONDecodeError:
-                                verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
+                            data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line)
+                            delta: _SSEStopDelta = {} if data is None else data.get("delta", {})
+                            stop_reason: object = delta.get("stop_reason")
+                            if stop_reason is not None:
+                                return True
 
                 except Exception as e:
                     verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e)

litellm/llms/azure_ai/image_generation/mai_transformation.py

@@ -1,7 +1,12 @@
-from typing import TYPE_CHECKING, Any, Final
+import math
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
 
 import httpx
+from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
+from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.image_generation.transformation import (
@@ -18,6 +23,67 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
 
+_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float)
+
+
+def _token_count_or_none(value: object) -> int | float | None:
+    if value is None:
+        return None
+    try:
+        count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value)
+    except ValidationError:
+        verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent")
+        return None
+    if not math.isfinite(count):
+        verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent")
+        return None
+    return count
+
+
+_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)]
+
+
+@with_config(ConfigDict(extra="allow"))
+class _MAIImageUsageDetails(TypedDict, total=False):
+    text_tokens: ReadOnly[NotRequired[_TokenCount]]
+    image_tokens: ReadOnly[NotRequired[_TokenCount]]
+
+
+@with_config(ConfigDict(extra="allow"))
+class _MAIImageUsage(TypedDict, total=False):
+    input_tokens_details: ReadOnly[NotRequired[object]]
+    num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]]
+    num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]]
+    input_tokens: ReadOnly[NotRequired[_TokenCount]]
+    output_tokens: ReadOnly[NotRequired[_TokenCount]]
+    num_output_tokens: ReadOnly[NotRequired[_TokenCount]]
+    output_image_tokens: ReadOnly[NotRequired[_TokenCount]]
+    total_tokens: ReadOnly[NotRequired[_TokenCount]]
+
+
+@with_config(ConfigDict(extra="allow"))
+class _MAIImageResponse(TypedDict, total=False):
+    usage: ReadOnly[NotRequired[object]]
+
+
+_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails)
+_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage)
+_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None:
+    if usage is None:
+        return None
+    try:
+        return _MAI_RAW_USAGE_ADAPTER.validate_python(usage)
+    except ValidationError:
+        verbose_logger.warning("MAI image usage is not an object, counting it as absent")
+        return None
+
+
+_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse)
+
+
 class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
     """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
 
@@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
         return "maiimage" in model_normalized
 
     @staticmethod
-    def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]:
+    def normalize_mai_image_usage(usage: object) -> Mapping[str, object]:
         """Map Azure MAI usage fields to OpenAI ImageUsage schema."""
-        if usage is None:
+        raw_usage: Final = _raw_usage_or_none(usage)
+        if raw_usage is None:
             return {
                 "input_tokens": 0,
                 "input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
@@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
                 "total_tokens": 0,
             }
 
-        normalized_usage: Final = dict(usage)
-        input_tokens_details = normalized_usage.get("input_tokens_details")
-        if not isinstance(input_tokens_details, dict):
-            input_tokens_details = {}
+        normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage)
+        input_tokens_details_value: Final = normalized_usage.get("input_tokens_details")
+        input_tokens_details: Final[_MAIImageUsageDetails] = (
+            _MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value)
+            if isinstance(input_tokens_details_value, dict)
+            else {}
+        )
 
         text_tokens = normalized_usage.get("num_input_text_tokens")
         if text_tokens is None:
@@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
         if total_tokens is None:
             total_tokens = input_tokens + output_tokens
 
-        normalized_usage.update(
-            {
-                "input_tokens": input_tokens,
-                "input_tokens_details": {
-                    "image_tokens": image_tokens,
-                    "text_tokens": text_tokens,
-                },
-                "output_tokens": output_tokens,
-                "total_tokens": total_tokens,
-            }
-        )
-        return normalized_usage
+        return {
+            **raw_usage,
+            "input_tokens": input_tokens,
+            "input_tokens_details": {
+                "image_tokens": image_tokens,
+                "text_tokens": text_tokens,
+            },
+            "output_tokens": output_tokens,
+            "total_tokens": total_tokens,
+        }
 
     def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
         return ["n", "size"]
@@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
         json_mode: bool | None = None,
     ) -> ImageResponse:
         try:
-            response: Final = raw_response.json()
+            response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json())
         except Exception:
             raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
 
-        if "usage" in response:
-            response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
+        response: Final = (
+            {
+                **response_json,
+                "usage": self.normalize_mai_image_usage(response_json["usage"]),
+            }
+            if "usage" in response_json
+            else dict(response_json)
+        )
 
         logging_obj.post_call(
             input=request_data.get("prompt", ""),

litellm/llms/bedrock/embed/embedding.py

@@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Final, get_args, overload
 
 import httpx
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
@@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config
 from .cohere_transformation import BedrockCohereEmbeddingConfig
 from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
@@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return response.json()
+        return _JSON_OBJECT.validate_python(response.json())
 
     async def _make_async_call(
         self,
@@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return response.json()
+        return _JSON_OBJECT.validate_python(response.json())
 
     def _transform_response(
         self,
@@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM):
 
         # Parse response
         if response.status_code == 200:
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         else:
             raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")
 

litellm/llms/bigmodel/chat/transformation.py

@@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
     def custom_llm_provider(self) -> str:
         return "bigmodel"
 
+    @property
+    def prompts_unsupported_json_schema(self) -> bool:
+        return False
+
     def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
         return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
 

litellm/llms/clf_ai_gateway/chat/transformation.py

@@ -0,0 +1,95 @@
+from typing import Final
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
+from litellm.secret_managers.main import get_secret_str
+
+SUPPORTED_OPENAI_PARAMS: Final = (
+    "stream",
+    "stream_options",
+    "frequency_penalty",
+    "presence_penalty",
+    "logit_bias",
+    "logprobs",
+    "top_logprobs",
+    "max_tokens",
+    "max_completion_tokens",
+    "n",
+    "stop",
+    "temperature",
+    "top_p",
+    "seed",
+    "response_format",
+    "tools",
+    "tool_choice",
+    "parallel_tool_calls",
+    "user",
+    "store",
+    "service_tier",
+)
+
+
+class ClfAiGatewayConfig(OpenAIGPTConfig):
+    @property
+    def custom_llm_provider(self) -> str | None:
+        return "clf_ai_gateway"
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: matches OpenAIGPTConfig's list return
+        supports_reasoning: Final = litellm.supports_reasoning(
+            model=model,
+            custom_llm_provider=self.custom_llm_provider,
+        )
+        extra: Final = ("reasoning_effort",) if supports_reasoning else ()
+        return [*SUPPORTED_OPENAI_PARAMS, *extra]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict[str, object],  # mutable-ok: OpenAIConfig.map_openai_params signature
+        optional_params: dict[str, object],  # mutable-ok: OpenAIConfig.map_openai_params signature
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: OpenAIConfig.map_openai_params signature
+        requested: Final = non_default_params.get("reasoning_effort")
+        requested_mapping: Final = as_str_mapping(requested)
+        effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested
+        declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway")
+        drop_enabled: Final = drop_params or litellm.drop_params is True
+        if requested_mapping is not None and effort is None and not drop_enabled:
+            raise UnsupportedParamsError(
+                message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop "
+                "unsupported openai params from the call, set `litellm.drop_params = True`",
+                llm_provider="clf_ai_gateway",
+                model=model,
+                status_code=400,
+            )
+        drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared
+        if drops_effort:
+            verbose_logger.warning(
+                "Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.",
+                effort,
+                model,
+                declared,
+            )
+        others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"}
+        normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others
+        forwarded: Final = others if drops_effort else normalized
+        mapped: Final[dict[str, object]] = super().map_openai_params(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # OpenAIGPTConfig types its params and result as bare dicts  # mutable-ok: parent returns a dict
+            non_default_params=forwarded,
+            optional_params=optional_params,
+            model=model,
+            drop_params=drop_params,
+        )
+        return mapped
+
+    def get_openai_compatible_provider_info(
+        self, api_base: str | None, api_key: str | None
+    ) -> tuple[str | None, str | None]:
+        resolved_api_base: Final = (
+            api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1"
+        )
+        dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY")
+        return resolved_api_base, dynamic_api_key

litellm/llms/edenai/audio_transcription/transformation.py

@@ -9,6 +9,7 @@ from collections.abc import Mapping
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
 from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
@@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object
 
 from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]:  # mutable-ok: httpx form data
     """LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a
@@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
     def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
         if "application/json" not in raw_response.headers.get("content-type", ""):
             return TranscriptionResponse(text=raw_response.text)
-        body: Final = raw_response.json()
+        body: Final = _JSON_OBJECT.validate_python(raw_response.json())
         response: Final[TranscriptionResponse] = convert_to_model_response_object(
             response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription"
         )

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         responses_so_far: Sequence["ModelResponseStream"],
         litellm_logging_obj: "LiteLLMLoggingObj | None",
     ) -> "ModelResponse":
-        """``stream_chunk_builder`` folds every choice of a stream into one index-0
-        choice, so the stream is rebuilt one choice index at a time (every chunk
-        kept, its choices narrowed to that index, so usage-only chunks still
-        count) and the rebuilt choices are stitched into one response, each
-        carrying the index the stream gave it."""
         choice_indices: Final = tuple(
             sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
         )

litellm/llms/openai/image_generation/gpt_transformation.py

@@ -1,6 +1,7 @@
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
@@ -9,6 +10,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
 from litellm.types.utils import ImageResponse
 from litellm.utils import convert_to_model_response_object
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
 
@@ -66,7 +69,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
         api_key: str | None = None,
         json_mode: bool | None = None,
     ) -> ImageResponse:
-        response: Final = raw_response.json()
+        response: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         stringified_response: Final = response
         ## LOGGING

litellm/llms/vertex_ai/batches/handler.py

@@ -15,8 +15,10 @@ from litellm.litellm_core_utils.url_utils import (
 )
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
+    MaskedHTTPStatusError,
     get_async_httpx_client,
     get_httpx_client,
+    mask_sensitive_info,
 )
 from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
@@ -81,6 +83,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str
     return response.json()
 
 
+def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError:
+    body: Final = mask_sensitive_info(error.response.text)
+    return MaskedHTTPStatusError(
+        httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response),
+        message=body,
+        text=body,
+    )
+
+
 class VertexAIBatchPrediction(VertexLLM):
     def __init__(self, gcs_bucket_name: str, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -166,11 +177,19 @@ class VertexAIBatchPrediction(VertexLLM):
                 headers=headers,
             )
 
-        response: Final = sync_handler.post(
-            url=api_base,
-            headers=headers,
-            data=json.dumps(vertex_batch_request),
-        )
+        try:
+            response: Final = sync_handler.post(
+                url=api_base,
+                headers=headers,
+                data=json.dumps(vertex_batch_request),
+            )
+        except httpx.HTTPStatusError as e:
+            litellm.verbose_logger.error(
+                "Vertex AI batch create failed: status=%s, body=%s",
+                e.response.status_code,
+                e.response.text[:1000],
+            )
+            raise _vertex_batch_create_error(e) from e
 
         payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
         _json_response: Final = payload_view["payload"]
@@ -266,13 +285,12 @@ class VertexAIBatchPrediction(VertexLLM):
                 data=json.dumps(vertex_batch_request),
             )
         except httpx.HTTPStatusError as e:
-            error_body: Final = e.response.text
             litellm.verbose_logger.error(
                 "Vertex AI batch create failed: status=%s, body=%s",
                 e.response.status_code,
-                error_body[:1000],
+                e.response.text[:1000],
             )
-            raise
+            raise _vertex_batch_create_error(e) from e
 
         payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
         _json_response: Final = payload_view["payload"]

litellm/llms/vertex_ai/batches/transformation.py

@@ -9,14 +9,41 @@ from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.llms.vertex_ai.common_utils import (
     VertexAIError,
+    VertexAIModelRoute,
     convert_vertex_datetime_to_openai_datetime,
+    get_vertex_ai_model_route,
 )
+from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
 from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
 from litellm.types.llms.vertex_ai import *
-from litellm.types.llms.vertex_ai import GenerateContentResponseBody
+from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider
 from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage
 
 _NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
+VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16"
+_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/"
+
+
+def is_vertex_anthropic_batch_model(model: str) -> bool:
+    """
+    Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the
+    way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/<model>` with rows in
+    Anthropic's Messages shape
+    """
+    bare_model: Final = model.rsplit("/", 1)[-1]
+    return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or (
+        get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS
+        and VertexPartnerProvider.claude.value in bare_model
+    )
+
+
+def as_json_object(value: object) -> Mapping[str, object] | None:
+    return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None
+
+
+def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool:
+    return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict)
 
 
 def _int_field(mapping: Mapping[str, object], key: str) -> int:
@@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object
     return isinstance(response_body.get("embedding"), dict)
 
 
+def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
+    """
+    Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the
+    request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty
+    response plus a `status` on failure, so a Claude input file read back is never mistaken for output
+    """
+    if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row):
+        return False
+    response: Final = as_json_object(vertex_output_row.get("response"))
+    return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool(
+        vertex_output_row.get("status")
+    )
+
+
 def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool:
-    return isinstance(row.get("request"), dict)
+    return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row)
 
 
 class NativeVertexBatchCostCalculator(Protocol):
@@ -72,6 +113,9 @@ def _native_vertex_row_usage(
     response_body: Mapping[str, object],
     calculate_usage: Callable[[GenerateContentResponseBody], Usage],
 ) -> Usage | None:
+    anthropic_usage: Final = as_json_object(response_body.get("usage"))
+    if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None:
+        return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None)
     if "usageMetadata" not in response_body:
         if not is_vertex_embedding_batch_output_response(response_body):
             return None
@@ -97,21 +141,24 @@ def native_vertex_batch_row_stats(
     """
     Usage and cost of one native Vertex predictions.jsonl row, a
     `{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}`
-    generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
-    embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`).
-    `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion`
-    does, else the wildcard name so explicit deployment prices still apply; a row without a response, a
-    generateContent row without `response.usageMetadata`, and a row whose response fails validation are
+    generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
+    embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a
+    `{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object.
+    `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude
+    rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response,
+    a generateContent row without `response.usageMetadata`, and a row whose response fails validation are
     None (failed).
     """
-    response_body: Final = row.get("response")
-    if not isinstance(response_body, dict):
+    response_body: Final = as_json_object(row.get("response"))
+    if response_body is None:
         return None
     usage: Final = _native_vertex_row_usage(response_body, calculate_usage)
     if usage is None:
         return None
     total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
-    model_version: Final = response_body.get("modelVersion")
+    model_version: Final = response_body.get(
+        "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
+    )
     deployment_model: Final = model_name if model_name and "*" not in model_name else None
     model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
     if model is None:

litellm/llms/vertex_ai/files/transformation.py

@@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required
 
 import litellm
 from litellm._uuid import uuid
+from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME
 from litellm.files.types import FileContentStreamingResult
 from litellm.files.utils import FilesAPIUtils
 from litellm.litellm_core_utils.cloud_storage_security import (
@@ -32,13 +33,14 @@ from litellm.litellm_core_utils.cloud_storage_security import (
     split_configured_cloud_bucket_name,
     validate_managed_cloud_file_id,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_data,
     extract_file_metadata,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY
 from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@@ -47,7 +49,14 @@ from litellm.llms.base_llm.files.transformation import (
     BaseFileUploadStream,
     LiteLLMLoggingObj,
 )
-from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count
+from litellm.llms.vertex_ai.batches.transformation import (
+    VERTEX_ANTHROPIC_VERSION,
+    as_json_object,
+    is_vertex_anthropic_batch_model,
+    is_vertex_anthropic_batch_output_response,
+    is_vertex_anthropic_batch_output_row,
+    vertex_embedding_prompt_token_count,
+)
 from litellm.llms.vertex_ai.common_utils import (
     convert_vertex_datetime_to_openai_datetime,
     get_vertex_ai_fine_tuned_endpoint_id,
@@ -59,6 +68,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
 from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
     transform_openai_input_gemini_embed_content,
 )
+from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
 from litellm.types.files import StreamingMediaUploadConfig
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -79,6 +89,7 @@ from litellm.types.utils import (
     ModelResponse,
     Usage,
 )
+from litellm.utils import get_non_default_completion_params, get_optional_params
 
 from ..common_utils import VertexAIError
 from ..vertex_llm_base import VertexBase
@@ -87,6 +98,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63
 _CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_"
 _VERTEX_BATCH_KEY_FIELD: Final = "key"
 _MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)")
+_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/")
 _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = (
     ("outputDimensionality", "output_dimensionality"),
     ("taskType", "task_type"),
@@ -225,10 +237,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str
     """
     Resolve the OpenAI `custom_id` for a Vertex batch output row.
 
-    Embedding rows carry it in the top-level `key` field that Vertex echoes back;
-    `generateContent` rows have no such field, so it is smuggled through request
-    labels instead (see `_set_litellm_batch_custom_id_labels`).
+    Claude rows carry it in their own top-level `custom_id` field and embedding rows in
+    the top-level `key` field that Vertex echoes back; `generateContent` rows have no
+    such field, so it is smuggled through request labels instead (see
+    `_set_litellm_batch_custom_id_labels`).
     """
+    anthropic_custom_id: Final = vertex_output_row.get("custom_id")
+    if anthropic_custom_id is not None:
+        return str(anthropic_custom_id)
     key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD)
     if key is not None:
         return unquote(str(key))
@@ -289,6 +305,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str,
     )
 
 
+def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
+    return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row(
+        vertex_output_row
+    )
+
+
 def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None:
     try:
         row: Final = _parse_vertex_batch_output_row(line.decode("utf-8"))
@@ -410,6 +432,37 @@ def _openai_batch_output_row(
     }
 
 
+def _vertex_anthropic_batch_output_row_to_openai(
+    custom_id: str,
+    response: object,
+    mock_httpx_response: httpx.Response,
+) -> _OpenAIBatchOutputRow:
+    """
+    One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response
+    """
+    anthropic_response: Final = as_json_object(response)
+    if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response):
+        return _openai_batch_output_row(
+            custom_id=custom_id,
+            error_code="vertex_ai_error",
+            error_message="Claude batch row has no message response",
+        )
+    try:
+        model_response: Final = VertexAIAnthropicConfig().transform_parsed_response(
+            completion_response=dict(anthropic_response),
+            raw_response=mock_httpx_response,
+            model_response=ModelResponse(),
+            json_mode=True,
+        )
+    except Exception as e:  # noqa: BLE001  # one malformed row must fail alone, not the whole output file
+        return _openai_batch_output_row(
+            custom_id=custom_id,
+            error_code="transformation_error",
+            error_message=f"Failed to transform response: {e}",
+        )
+    return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump())
+
+
 def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]:
     """
     Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch
@@ -684,21 +737,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
     )
 
 
+_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset(
+    ("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options")
+)
+_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"}
+_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = (
+    _CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"}
+) | {"drop_params"}
+
+
+def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType(
+        {key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS}
+    )
+
+
+def _names_the_response_format_tool(tool: object) -> bool:
+    match tool:
+        case {"function": {"name": str(name)}}:
+            return name == RESPONSE_FORMAT_TOOL_NAME
+        case _:
+            return False
+
+
+def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
+    custom_id: object,
+    messages: list[AllMessageValues],
+    request_params: Mapping[str, object],
+    model: str,
+    deployment_params: Mapping[str, object],
+) -> Mapping[str, object]:
+    """
+    Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with,
+    `{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}`
+    """
+    if custom_id is None:
+        raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it")
+    config: Final = VertexAIAnthropicConfig()
+    bare_model: Final = model.rsplit("/", 1)[-1]
+    params: Final = {**deployment_params, **request_params}
+    drop_params: Final = drops_unsupported_params(params.get("drop_params"))
+    chat_params: Final = {
+        **get_non_default_completion_params(params),
+        **{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS},
+    }
+    if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))):
+        raise ValueError(
+            f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, "
+            "and batch output cannot tell a user tool by that name apart, so rename it"
+        )
+    optional_params: Final[Mapping[str, object]] = get_optional_params(
+        model=bare_model,
+        custom_llm_provider="vertex_ai",
+        drop_params=drop_params,
+        messages=messages,
+        **{  # pyright: ignore[reportArgumentType]  # batch JSONL bodies are untyped; get_optional_params checks each param as chat does
+            key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS
+        },
+    )
+    request_state: Final[dict[str, object]] = {"drop_params": drop_params}
+    anthropic_request: Final[Mapping[str, object]] = config.transform_request(
+        model=bare_model,
+        messages=messages,
+        optional_params={
+            **{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")},
+            "anthropic_version": VERTEX_ANTHROPIC_VERSION,
+        },
+        litellm_params=request_state,
+        headers={},
+    )
+    renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY))
+    if renamed_tools:
+        raise ValueError(
+            f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, "
+            "and batch output cannot map a renamed tool back, so rename these tools"
+        )
+    extra_body: Final = as_json_object(optional_params.get("extra_body"))
+    request: Final[Mapping[str, object]] = (
+        anthropic_request
+        if extra_body is None
+        else {
+            **anthropic_request,
+            **config.transform_extra_body(
+                extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={}
+            ),
+        }
+    )
+    return {"custom_id": str(custom_id), "request": request}
+
+
 def _openai_batch_jsonl_entry_to_vertex_rows(
     openai_entry: dict[str, Any],
     map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
+    batch_model: str | None = None,
+    deployment_params: Mapping[str, object] = MappingProxyType({}),
 ) -> tuple[Mapping[str, object], ...]:
     """
     Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
 
+    `batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`,
+    which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise.
+
     jsonl body for vertex is {"request": <request_body>}
     Example Vertex jsonl
     {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}}
     """
+    entry_body: Final = openai_entry.get("body")
+    model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "")
+    is_anthropic_model: Final = is_vertex_anthropic_batch_model(model)
     if _is_embeddings_batch_entry(openai_entry):
+        if is_anthropic_model:
+            raise VertexAIError(
+                status_code=400,
+                message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')",
+            )
         return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry)
 
-    if not isinstance(openai_entry.get("body"), dict):
+    if not isinstance(entry_body, dict):
         raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object")
     openai_request_body: Final = openai_entry["body"]
     chat_request_body: Final = (
@@ -706,8 +861,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
         if _is_responses_batch_entry(openai_entry)
         else openai_request_body
     )
+    chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", []))
+    if is_anthropic_model:
+        return (
+            _openai_batch_jsonl_entry_to_vertex_anthropic_row(
+                custom_id=openai_entry.get("custom_id"),
+                messages=chat_messages,
+                request_params={
+                    key: value for key, value in chat_request_body.items() if key not in ("model", "messages")
+                },
+                model=model,
+                deployment_params=deployment_params,
+            ),
+        )
     vertex_request_body: Final = transform_request_body(
-        messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
+        messages=chat_messages,
         model=chat_request_body.get("model", ""),
         optional_params=map_openai_to_vertex_params(chat_request_body),
         custom_llm_provider="vertex_ai",
@@ -859,15 +1027,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
         self,
         openai_file_content: FileTypes,
         map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
+        batch_model: str | None = None,
+        deployment_params: Mapping[str, object] = MappingProxyType({}),
     ) -> None:
         self._openai_file_content = openai_file_content
         self._map_openai_to_vertex_params = map_openai_to_vertex_params
+        self._batch_model = batch_model
+        self._deployment_params = deployment_params
 
     def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
         first = True
         for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
             try:
-                vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
+                vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
+                    entry,
+                    self._map_openai_to_vertex_params,
+                    batch_model=self._batch_model,
+                    deployment_params=self._deployment_params,
+                )
             except Exception as e:
                 raise VertexAIError(
                     status_code=400,
@@ -923,16 +1100,31 @@ class _RawFileUploadStream(BaseFileUploadStream):
         return _iter_raw_file_chunks(self._file_content)
 
 
+def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None:
+    model: Final = litellm_params.get("model")
+    return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None
+
+
+def _publisher_model_path(raw_model: str) -> str:
+    if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model):
+        return raw_model
+    publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google"
+    return f"publishers/{publisher}/models/{raw_model}"
+
+
 def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str:
     endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model)
-    model_path: Final = (
-        f"endpoints/{endpoint_id}"
-        if endpoint_id is not None
-        else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}")
-    )
-    safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model")
+    model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model)
     prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX
-    return f"{prefix}{safe_model_path}/{uuid.uuid4()}"
+    return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}"
+
+
+def _safe_managed_model_path(model_path: str) -> str:
+    if not is_vertex_anthropic_batch_model(model_path):
+        return sanitize_cloud_object_path(model_path, fallback="model")
+    return "@".join(
+        sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@")
+    )
 
 
 class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
@@ -1036,10 +1228,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
             fallback_filename="file",
         )
 
-    def _get_configured_bucket_name(self, litellm_params: dict) -> str:
+    def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str:
+        configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name")
         bucket_name: Final = (
-            litellm_params.get("gcs_bucket_name")
-            or litellm_params.get("bucket_name")
+            (configured_bucket_name if isinstance(configured_bucket_name, str) else None)
             or os.getenv("GCS_BATCH_BUCKET_NAME")
             or os.getenv("GCS_BUCKET_NAME")
         )
@@ -1053,7 +1245,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
         api_key: str | None,
         model: str,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
         data: CreateFileRequest,
     ) -> str:
         """
@@ -1077,11 +1269,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
             raise ValueError("file is required")
         if purpose is None:
             raise ValueError("purpose is required")
-        configured_model: Final = litellm_params.get("model")
         object_name = self.get_object_name(
             file_data,
             purpose,
-            deployment_model=configured_model if isinstance(configured_model, str) else None,
+            deployment_model=_deployment_batch_model(litellm_params),
             passthrough=is_passthrough_batch_upload(data, litellm_params),
         )
         if object_prefix:
@@ -1138,7 +1329,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
         model: str,
         create_file_data: CreateFileRequest,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
     ) -> bytes | str | dict:
         """
         2 Cases:
@@ -1170,6 +1361,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
                     body_stream=_OpenAIToVertexBatchUploadStream(
                         file_data,
                         partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
+                        batch_model=_deployment_batch_model(litellm_params),
+                        deployment_params=_claude_batch_deployment_params(litellm_params),
                     ),
                     content_type="application/json",
                 )
@@ -1220,7 +1413,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
     def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException:
         return VertexAIError(status_code=status_code, message=error_message, headers=headers)
 
-    def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]:
+    def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]:
         """
         Validate a managed GCS file_id and return (bucket, url-encoded-object-path).
         """
@@ -1238,7 +1431,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
         self,
         file_id: str,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
     ) -> tuple[str, dict]:
         bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
         url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
@@ -1268,7 +1461,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
         self,
         file_id: str,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
     ) -> tuple[str, dict]:
         bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
         url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
@@ -1311,7 +1504,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
         self,
         file_content_request,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
     ) -> tuple[str, dict]:
         file_id: Final = file_content_request.get("file_id", "")
         bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
@@ -1408,7 +1601,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
                 headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}),
             )
 
-        if not _is_vertex_generate_content_batch_output_row(first_row):
+        if not _is_vertex_chat_batch_output_row(first_row):
             return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers)
 
         return FileContentStreamingResult(
@@ -1492,10 +1685,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
             # first line is not valid UTF-8/JSON) raises and falls through to the
             # passthrough below, leaving the content untouched.
             first_row: Final = _parse_vertex_batch_output_row(first_line)
-            if not (
-                _is_vertex_embeddings_batch_output_row(first_row)
-                or _is_vertex_generate_content_batch_output_row(first_row)
-            ):
+            if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)):
                 return content
 
             context: Final = _new_vertex_batch_output_row_transform_context()
@@ -1560,6 +1750,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
                 error_message=status,
             )
 
+        if is_vertex_anthropic_batch_output_row(vertex_output):
+            return _vertex_anthropic_batch_output_row_to_openai(
+                custom_id=custom_id,
+                response=vertex_output.get("response"),
+                mock_httpx_response=mock_httpx_response,
+            )
+
         # Transform successful response using existing transformation
         vertex_response: Final = vertex_output.get("response", {})
 

litellm/llms/zai/chat/transformation.py

@@ -1,11 +1,43 @@
+from collections.abc import Mapping
 from typing import Final
 
+from typing_extensions import TypeIs  # noqa: TID251  # TypeIs reaches typing only on 3.13
+
+import litellm
+from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt
 from litellm.secret_managers.main import get_secret_str
-from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage
+from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4"
+TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"})
+
+
+def _is_json_object(
+    value: object,
+) -> TypeIs[Mapping[str, object]]:  # guard-ok: cost-map rows and request params are JSON
+    return isinstance(value, Mapping)
+
+
+def _json_field(value: object, key: str) -> object:
+    return value.get(key) if _is_json_object(value) else None
+
+
+def _exceeds_output_limit(model: str, max_tokens: object) -> bool:
+    info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai")
+    limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens")
+    return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit
+
+
+def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None:
+    if _json_field(response_format, "type") != "json_schema":
+        return None
+    if litellm.supports_response_schema(model=model, custom_llm_provider="zai"):
+        return None
+    schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema")
+    return schema if _is_json_object(schema) else None
 
 
 class ZAIChatConfig(OpenAIGPTConfig):
@@ -13,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig):
     def custom_llm_provider(self) -> str | None:
         return "zai"
 
+    @property
+    def prompts_unsupported_json_schema(self) -> bool:
+        return True
+
     def _get_openai_compatible_provider_info(
         self, api_base: str | None, api_key: str | None
     ) -> tuple[str | None, str | None]:
@@ -40,9 +76,11 @@ class ZAIChatConfig(OpenAIGPTConfig):
         # GLM/ZAI supports cache_control, so return messages and tools unchanged
         return messages, tools
 
-    def get_supported_openai_params(self, model: str) -> list:
+    def get_supported_openai_params(self, model: str) -> list[str]:
         base_params: Final = [
             "max_tokens",
+            "max_completion_tokens",
+            "response_format",
             "stream",
             "stream_options",
             "temperature",
@@ -52,8 +90,6 @@ class ZAIChatConfig(OpenAIGPTConfig):
             "tool_choice",
         ]
 
-        import litellm
-
         try:
             if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider):
                 base_params.append("thinking")
@@ -61,3 +97,53 @@ class ZAIChatConfig(OpenAIGPTConfig):
             pass
 
         return base_params
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        supported_params: Final = self.get_supported_openai_params(model)
+        sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
+        max_completion_tokens: Final = sent.get("max_completion_tokens")
+        falls_back_to_max_tokens: Final = max_completion_tokens is None or (
+            (litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens)
+        )
+        max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens
+        return {
+            **optional_params,
+            **{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
+            **({} if max_tokens is None else {"max_tokens": max_tokens}),
+        }
+
+    def transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: OpenAIGPTConfig.transform_request signature
+        optional_params: dict[str, object],  # mutable-ok: OpenAIGPTConfig.transform_request signature
+        litellm_params: dict[str, object],  # mutable-ok: OpenAIGPTConfig.transform_request signature
+        headers: dict[str, str],  # mutable-ok: OpenAIGPTConfig.transform_request signature
+    ) -> dict[str, object]:  # mutable-ok: inherited contract returns the request body
+        schema: Final = (
+            _schema_without_native_support(model, optional_params.get("response_format"))
+            if self.prompts_unsupported_json_schema
+            else None
+        )
+        schema_message: Final = (
+            ()
+            if schema is None
+            else (
+                ChatCompletionUserMessage(
+                    role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai")
+                ),
+            )
+        )
+        return super().transform_request(
+            model,
+            [*messages, *schema_message],
+            optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}},
+            litellm_params,
+            headers,
+        )

litellm/provider_endpoints_support_backup.json

@@ -827,6 +827,32 @@
         "generateContent": false
       }
     },
+    "clf_ai_gateway": {
+      "display_name": "CLF AI Gateway (`clf_ai_gateway`)",
+      "url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
+    },
     "cloudflare": {
       "display_name": "Cloudflare AI Workers (`cloudflare`)",
       "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers",

litellm/proxy/_types.py

@@ -3177,11 +3177,18 @@ from litellm.types.proxy.auth.user_api_key_auth import (
 )
 
 
+class AdminMCPPermissions(LiteLLMPydanticObjectBase):
+    model_config = ConfigDict(frozen=True)
+    tools: tuple[str, ...]
+    action_allowed: bool | None = None
+
+
 class UserInfoResponse(LiteLLMPydanticObjectBase):
     user_id: str | None
     user_info: dict | BaseModel | None
     keys: list
     teams: list
+    admin_mcp_permissions: AdminMCPPermissions | None = None
 
 
 class UserInfoV2Response(LiteLLMPydanticObjectBase):

litellm/proxy/admin_mcp.py

@@ -1,20 +1,21 @@
 import os
 import re
-from collections.abc import AsyncGenerator, Mapping
+from collections.abc import AsyncGenerator, Iterator, Mapping
 from contextlib import asynccontextmanager
 from contextvars import ContextVar
 from typing import Final
-from urllib.parse import urlsplit
+from urllib.parse import unquote, urlsplit
 
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.routing import APIRoute
 from pydantic import TypeAdapter
 from starlette.datastructures import Headers
-from starlette.requests import Request
 from starlette.routing import Mount
 from starlette.types import ASGIApp, Receive, Scope, Send
 
 from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
-from litellm.proxy._types import SpecialHeaders
+from litellm.proxy._types import AdminMCPPermissions, SpecialHeaders, UserAPIKeyAuth
+from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.middleware.admission_control_middleware import ADMISSION_LEASE_SCOPE_KEY
 
 _REQUEST_HEADERS: Final = frozenset(
@@ -58,6 +59,91 @@ _RESERVED_KEY_HEADERS: Final = (
 _SETTINGS: Final = TypeAdapter(Mapping[str, object])
 _IDENTITY_MAPPINGS: Final = TypeAdapter(tuple[dict[str, object], ...] | dict[str, object] | None)
 _OAUTH_MAPPINGS: Final = TypeAdapter(dict[str, str])
+_ROUTE_GRANTS: Final = TypeAdapter(tuple[str, ...])
+
+
+def _admin_route_allowed(auth: UserAPIKeyAuth, request: Request, method: str, path: str) -> bool:
+    target: Final = Request({**request.scope, "method": method, "path": path, "raw_path": path.encode()})
+    try:
+        RouteChecks.should_call_route(route=path, valid_token=auth, request=target)
+    except HTTPException:
+        return False
+    return True
+
+
+def _admin_routes(request: Request) -> Iterator[tuple[tuple[str, str], APIRoute]]:
+    app: Final = request.app
+    if not isinstance(app, FastAPI):
+        return
+    for route in app.routes:
+        if not isinstance(route, APIRoute):
+            continue
+        for method in route.methods:
+            yield (method, route.path_format), route
+
+
+def _admin_grant_paths(route: APIRoute, grant: str) -> Iterator[str]:
+    if route.path_regex.fullmatch(grant) is not None:
+        yield grant
+    prefix: Final = grant[:-1] if grant.endswith("*") else grant + "/"
+    representative: Final = re.sub(r"\{[^}]+\}", "_", route.path_format)
+    # A matching suffix supplies the rest of a concrete route for prefix grants,
+    # including grants ending inside a static segment or a path parameter.
+    for candidate in (prefix + representative[offset:] for offset in range(len(representative) + 1)):
+        if route.path_regex.fullmatch(candidate) is not None:
+            yield candidate
+
+
+def _admin_grant_allowed(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute, grant: str) -> bool:
+    return any(_admin_route_allowed(auth, request, method, path) for path in _admin_grant_paths(route, grant))
+
+
+def _admin_tool_visible(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute) -> bool:
+    try:
+        RouteChecks.check_route_enabled(route.path)
+    except HTTPException:
+        return False
+    if _admin_route_allowed(auth, request, method, route.path):
+        return True
+    grants: Final = _ROUTE_GRANTS.validate_python(getattr(auth, "allowed_routes", None) or ())
+    return bool(route.param_convertors) and any(
+        _admin_grant_allowed(auth, request, method, route, grant) for grant in grants if grant.startswith("/")
+    )
+
+
+def get_admin_mcp_permissions(
+    auth: UserAPIKeyAuth, request: Request, tool: str | None = None, path: str | None = None
+) -> AdminMCPPermissions:
+    try:
+        from litellm_admin_mcp.catalog import (  # pyright: ignore[reportMissingTypeStubs]  # connector has no py.typed marker
+            BY_NAME,
+            OPERATIONS,
+        )
+    except ImportError as exc:
+        raise HTTPException(
+            status_code=503, detail="Admin MCP permissions require the admin-mcp dependency group (Python 3.12+)"
+        ) from exc
+
+    routes: Final = dict(_admin_routes(request))
+    allowed: Final = tuple(
+        operation.name
+        for operation in OPERATIONS
+        if (route := routes.get((operation.method, operation.path))) is not None
+        and _admin_tool_visible(auth, request, operation.method, route)
+    )
+    if tool is None:
+        return AdminMCPPermissions(tools=allowed)
+    operation: Final = BY_NAME.get(tool)
+    if operation is None or tool not in allowed:
+        return AdminMCPPermissions(tools=allowed, action_allowed=False)
+    if path is None:
+        return AdminMCPPermissions(tools=allowed, action_allowed=True)
+    registered: Final = routes[(operation.method, operation.path)]
+    decoded_path: Final = unquote(path)
+    action_allowed: Final = registered.path_regex.fullmatch(decoded_path) is not None and _admin_route_allowed(
+        auth, request, operation.method, decoded_path
+    )
+    return AdminMCPPermissions(tools=allowed, action_allowed=action_allowed)
 
 
 def _configured_key_header() -> bytes | None:

litellm/proxy/auth/oauth2_check.py

@@ -3,6 +3,7 @@ import os
 from typing import Final, cast
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.custom_httpx.http_handler import (
@@ -11,6 +12,8 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
 
+_TOKEN_INFO_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 class OAuth2TokenValidationError(ValueError):
     """The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it
@@ -184,7 +187,7 @@ class Oauth2Handler:
             response.raise_for_status()
 
             # If we get here, the request was successful
-            data = response.json()
+            data = _TOKEN_INFO_RESPONSE.validate_python(response.json())
 
             verbose_proxy_logger.debug(
                 "Oauth2 token validation for token=%s, response from endpoint=%s",

litellm/proxy/auth/route_checks.py

@@ -111,14 +111,7 @@ _TOKEN_COUNTING_ACTION: Final = "countTokens"
 
 class RouteChecks:
     @staticmethod
-    def should_call_route(
-        route: str,
-        valid_token: UserAPIKeyAuth,
-        request: Request | None = None,
-    ):
-        """
-        Check if management route is disabled and raise exception
-        """
+    def check_route_enabled(route: str) -> None:
         try:
             from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks
 
@@ -128,6 +121,16 @@ class RouteChecks:
         except Exception:
             pass
 
+    @staticmethod
+    def should_call_route(
+        route: str,
+        valid_token: UserAPIKeyAuth,
+        request: Request | None = None,
+    ):
+        """
+        Check if management route is disabled and raise exception
+        """
+        RouteChecks.check_route_enabled(route)
         # Check if Virtual Key is allowed to call the route - Applies to all Roles
         RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token, request=request)
         return True

litellm/proxy/client/cli/commands/models.py

@@ -1,7 +1,7 @@
 # stdlib imports
 import re
 from collections import defaultdict
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, Literal
@@ -61,6 +61,10 @@ def _get_model_info_obj_from_yaml(model: dict[str, Any]) -> ModelYamlInfo:
     )
 
 
+def _str_field(model: Mapping[str, object], key: str, default: str) -> str:
+    return str(model.get(key, default))
+
+
 def format_iso_datetime_str(iso_datetime_str: str | None) -> str:
     """Format an ISO format datetime string to human-readable date with minute resolution."""
     if not iso_datetime_str:
@@ -142,10 +146,10 @@ def list_models(ctx: click.Context, output_format: Literal["table", "json"]) ->
                 created = int(created)
 
             table.add_row(
-                str(model.get("id", "")),
-                str(model.get("object", "model")),
+                _str_field(model, "id", ""),
+                _str_field(model, "object", "model"),
                 (format_timestamp(created) if isinstance(created, int) else format_iso_datetime_str(created)),
-                str(model.get("owned_by", "")),
+                _str_field(model, "owned_by", ""),
             )
 
         rich.print(table)

litellm/proxy/client/users.py

@@ -2,9 +2,13 @@ from collections.abc import Iterable, Mapping
 from typing import Any, Final
 
 import requests
+from pydantic import ConfigDict, TypeAdapter
 
 from .exceptions import NotFoundError, UnauthorizedError
 
+_JSON_VALUE: Final = TypeAdapter(object, config=ConfigDict(strict=True, hide_input_in_errors=True))
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 class UsersManagementClient:
     def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
@@ -39,7 +43,7 @@ class UsersManagementClient:
         if response.status_code == 404:
             raise NotFoundError(response.text)
         response.raise_for_status()
-        return response.json()
+        return _JSON_OBJECT.validate_python(response.json())
 
     def get_user_v2(self, user_id: str | None = None) -> dict[str, object]:
         """Get user info v2 - lightweight, returns only user object (GET /v2/user/info)"""
@@ -51,7 +55,7 @@ class UsersManagementClient:
         if response.status_code == 404:
             raise NotFoundError(response.text)
         response.raise_for_status()
-        return response.json()
+        return _JSON_OBJECT.validate_python(response.json())
 
     def create_user(self, user_data: Mapping[str, object]) -> dict[str, object]:
         """Create a new user (POST /user/new)"""
@@ -60,9 +64,9 @@ class UsersManagementClient:
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         response.raise_for_status()
-        return response.json()
+        return _JSON_OBJECT.validate_python(response.json())
 
-    def delete_user(self, user_ids: list[str]) -> dict[str, object]:
+    def delete_user(self, user_ids: list[str]) -> object:
         """Delete users (POST /user/delete)"""
         url: Final = f"{self.base_url}/user/delete"
         response: Final = requests.post(
@@ -71,4 +75,4 @@ class UsersManagementClient:
         if response.status_code == 401:
             raise UnauthorizedError(response.text)
         response.raise_for_status()
-        return response.json()
+        return _JSON_VALUE.validate_python(response.json())

litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py

@@ -16,6 +16,10 @@ unknown impl value raises, which the proxy converts to the fail-closed 503.
 """
 
 import os
+from typing import Final
+
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy.management_helpers.team_metadata_validation import (
@@ -23,12 +27,20 @@ from litellm.proxy.management_helpers.team_metadata_validation import (
     TeamMetadataValidationResult,
 )
 from litellm.types.llms.custom_http import httpxSpecialProvider
-from typing import Final
 
 ALLOWED_COST_CENTERS: Final = frozenset({"CC-1001", "CC-1002"})
 CLOSED_PORT_URL: Final = "http://127.0.0.1:9/validate"
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CostCenterReply(TypedDict):
+    ok: NotRequired[ReadOnly[object]]
+    reason: NotRequired[ReadOnly[str | None]]
+
+
+_COST_CENTER_REPLY: Final = TypeAdapter(_CostCenterReply)
+
+
 async def _validate_allowlist(payload: TeamMetadataValidationPayload) -> TeamMetadataValidationResult:
     cost_center: Final = payload.metadata.get("cost_center")
     if cost_center is None:
@@ -54,7 +66,7 @@ async def _validate_via_http(payload: TeamMetadataValidationPayload, service_url
         },
         timeout=2.0,
     )
-    body: Final = response.json()
+    body: Final = _COST_CENTER_REPLY.validate_python(response.json())
     if body.get("ok") is True:
         return TeamMetadataValidationResult(valid=True)
     return TeamMetadataValidationResult(

litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py

@@ -22,6 +22,7 @@ import os
 from collections.abc import AsyncIterator, Mapping, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
+from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
@@ -120,6 +121,11 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
     """Base-class constructor options this guardrail forwards untouched to CustomGuardrail."""
 
 
+def _streamed_choice_index(choice: object) -> int:
+    index: Final[object] = getattr(choice, "index", None)
+    return index if isinstance(index, int) else 0
+
+
 class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
     """
     Cisco AI Defense guardrail integration.
@@ -585,29 +591,44 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
 
     @classmethod
     def _extract_streaming_chunk_scan_text(cls, chunks: Sequence[object]) -> str:
+        streamed_choices: Final = tuple(
+            choice
+            for choice in chain.from_iterable(getattr(chunk, "choices", None) or () for chunk in chunks)
+            if getattr(choice, "delta", None) is not None
+        )
+        choice_indices: Final = sorted(frozenset(map(_streamed_choice_index, streamed_choices)))
+        choice_texts: Final = (
+            cls._extract_streamed_deltas_scan_text(
+                tuple(
+                    getattr(choice, "delta", None)
+                    for choice in streamed_choices
+                    if _streamed_choice_index(choice) == index
+                )
+            )
+            for index in choice_indices
+        )
+        return " ".join(text for text in choice_texts if text)
+
+    @classmethod
+    def _extract_streamed_deltas_scan_text(cls, deltas: Sequence[object]) -> str:
         original_text = ""
         argument_text = ""
-        for chunk in chunks:
-            choices = getattr(chunk, "choices", None) or []
-            for c in choices:
-                delta: object | None = getattr(c, "delta", None)
-                if delta is None:
-                    continue
-                text = getattr(delta, "content", None)
-                if isinstance(text, str):
-                    original_text += text
-                reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta))
-                if reasoning_text:
-                    original_text += reasoning_text
-                for tc in getattr(delta, "tool_calls", None) or []:
-                    args = cls._extract_tool_call_arguments(tc)
-                    if args:
-                        argument_text += args
-                fc: object | None = getattr(delta, "function_call", None)
-                if fc is not None:
-                    args = cls._extract_function_call_arguments(fc)
-                    if args:
-                        argument_text += args
+        for delta in deltas:
+            text = getattr(delta, "content", None)
+            if isinstance(text, str):
+                original_text += text
+            reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta))
+            if reasoning_text:
+                original_text += reasoning_text
+            for tc in getattr(delta, "tool_calls", None) or []:
+                args = cls._extract_tool_call_arguments(tc)
+                if args:
+                    argument_text += args
+            fc: object | None = getattr(delta, "function_call", None)
+            if fc is not None:
+                args = cls._extract_function_call_arguments(fc)
+                if args:
+                    argument_text += args
         return " ".join(part for part in (original_text, argument_text) if part)
 
     # ------------------------------------------------------------------

litellm/proxy/guardrails/guardrail_hooks/noma/noma.py

@@ -246,20 +246,38 @@ class NomaGuardrail(CustomGuardrail):
     ) -> str | None:
         """Shared logic for processing LLM response checks"""
 
-        start_time: Final = datetime.now()
-        extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data)
-
         if not isinstance(response, litellm.ModelResponse):
             return None
 
-        content = None
-        for choice in response.choices:
-            if isinstance(choice, litellm.Choices) and choice.message.content:
-                content = choice.message.content
-                break
+        checked_contents: Final = tuple(
+            [
+                await self._process_choice_check(request_data, response, position, content, user_auth, event_type)
+                for position, content in self._choice_texts(response)
+            ]
+        )
+        return next(iter(checked_contents), None)
 
-        if not content or not isinstance(content, str):
-            return None
+    @staticmethod
+    def _choice_texts(response: litellm.ModelResponse) -> tuple[tuple[int, str], ...]:
+        return tuple(
+            (position, choice.message.content)
+            for position, choice in enumerate(response.choices)
+            if isinstance(choice, litellm.Choices)
+            and isinstance(choice.message.content, str)
+            and choice.message.content
+        )
+
+    async def _process_choice_check(
+        self,
+        request_data: dict[str, object],
+        response: litellm.ModelResponse,
+        position: int,
+        content: str,
+        user_auth: UserAPIKeyAuth,
+        event_type: GuardrailEventHooks | None,
+    ) -> str:
+        start_time: Final = datetime.now()
+        extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data)
 
         payload: Final = {
             "input": [
@@ -306,7 +324,7 @@ class NomaGuardrail(CustomGuardrail):
             anonymized_content: Final = self._extract_anonymized_content(response_json, ASSISTANT_ROLE)
             if anonymized_content:
                 # Replace the LLM response content with anonymized version
-                self._replace_llm_response_content(response, anonymized_content)
+                self._replace_llm_response_content(response, anonymized_content, position)
                 verbose_proxy_logger.debug("Noma guardrail anonymized LLM response: %s", anonymized_content)
                 return anonymized_content
 
@@ -486,21 +504,14 @@ class NomaGuardrail(CustomGuardrail):
                 messages[i]["content"] = anonymized_content
                 break
 
-    def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str):
-        """
-        Replace the LLM response content with anonymized version.
-
-        Args:
-            response: The original LLM response
-            anonymized_content: The anonymized content to replace with
-        """
+    def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str, position: int):
+        """Replace the content of the choice at ``position`` with its anonymized version."""
         if not isinstance(response, litellm.ModelResponse):
             return
 
-        # Replace content in all choices
-        for choice in response.choices:
-            if isinstance(choice, litellm.Choices) and choice.message.content:
-                choice.message.content = anonymized_content
+        choice: Final = response.choices[position]
+        if isinstance(choice, litellm.Choices):
+            choice.message.content = anonymized_content
 
     async def _check_user_message_background(
         self,
@@ -815,6 +826,7 @@ class NomaGuardrail(CustomGuardrail):
         )
 
         if isinstance(assembled_model_response, ModelResponse):
+            texts_before_check: Final = self._choice_texts(assembled_model_response)
             try:
                 processed_response: Final = await self._check_llm_response(
                     request_data,
@@ -828,7 +840,11 @@ class NomaGuardrail(CustomGuardrail):
                 if self.block_failures:
                     raise
                 verbose_proxy_logger.error("Noma streaming post-call hook failed: %s", e)
-                for chunk in all_chunks:
+                if self._choice_texts(assembled_model_response) == texts_before_check:
+                    for chunk in all_chunks:
+                        yield chunk
+                    return
+                async for chunk in MockResponseIterator(model_response=assembled_model_response):
                     yield chunk
                 return
 

litellm/proxy/guardrails/guardrail_hooks/tool_permission.py

@@ -36,6 +36,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
 )
 from litellm.types.utils import (
     CallTypesLiteral,
+    ChatCompletionMessageCustomToolCall,
     ChatCompletionMessageToolCall,
     Choices,
     Function,
@@ -744,6 +745,14 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         return ToolResult(tool_use_id=tool_call.id, content=error_message, is_error=True)
 
+    @staticmethod
+    def _judged_tool_call(
+        tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall,
+    ) -> tuple[str | None, ...]:
+        if isinstance(tool_call, ChatCompletionMessageCustomToolCall):
+            return (tool_call.id, tool_call.type, tool_call.custom.name, tool_call.custom.input)
+        return (tool_call.id, tool_call.type, tool_call.function.name, tool_call.function.arguments)
+
     def _modify_response_with_permission_errors(
         self,
         response: ModelResponse,
@@ -761,11 +770,10 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tools))
 
-        # Create a mapping of tool_use_id to error result
-        error_results: Final = {}
-        for tool_use, error in denied_tools:
-            error_result = self._create_permission_error_result(tool_use, error)
-            error_results[tool_use.id] = error_result
+        error_results: Final = {
+            self._judged_tool_call(tool_use): self._create_permission_error_result(tool_use, error)
+            for tool_use, error in denied_tools
+        }
 
         # Modify the response content
         for choice_index, choice in enumerate(response.choices):
@@ -775,9 +783,8 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
                 # Rewrite tool_calls
                 for tool_call in choice.message.tool_calls or []:
-                    tool_call_id = tool_call.id
-                    if tool_call_id in error_results:
-                        error_result = error_results[tool_call_id]
+                    error_result = error_results.get(self._judged_tool_call(tool_call))
+                    if error_result is not None:
                         error_messages.append(error_result.content)
                     else:
                         filtered_tool_calls.append(tool_call)
@@ -788,7 +795,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
                     getattr(choice.message, "function_call", None), choice_index
                 )
                 if legacy_tool_call is not None:
-                    legacy_error_result = error_results.get(legacy_tool_call.id)
+                    legacy_error_result = error_results.get(self._judged_tool_call(legacy_tool_call))
                     if legacy_error_result is not None:
                         choice.message.function_call = None
                         error_messages.append(legacy_error_result.content)

litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py

@@ -300,7 +300,7 @@ class VigilGuardGuardrail(CustomGuardrail):
     def _build_output(
         inputs: GenericGuardrailAPIInputs,
         final_texts: list[str],
-        final_tool_calls: Any,
+        final_tool_calls: _ToolCalls | None,
     ) -> GenericGuardrailAPIInputs:
         # When nothing was changed, return the input shape verbatim so the guardrail
         # logs "allow" rather than "mask". When a text or a tool-call argument was
@@ -315,7 +315,7 @@ class VigilGuardGuardrail(CustomGuardrail):
             guardrailed["images"] = inputs["images"]
         if "tools" in inputs:
             guardrailed["tools"] = inputs["tools"]
-        if tool_calls_changed:
+        if tool_calls_changed and final_tool_calls is not None:
             guardrailed["tool_calls"] = final_tool_calls
         return guardrailed
 

litellm/proxy/hooks/max_budget_per_session_limiter.py

@@ -18,6 +18,8 @@ import logging
 import os
 from typing import TYPE_CHECKING, Any, Final
 
+from pydantic import ConfigDict, TypeAdapter
+
 from litellm import DualCache
 from litellm._internal_context import with_service_target
 from litellm._logging import verbose_proxy_logger
@@ -36,6 +38,14 @@ else:
     InternalUsageCache = Any
 
 
+_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter(
+    str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+_OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter(
+    str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
+
 # Redis Lua script for atomic float increment with TTL.
 # INCRBYFLOAT returns the new value as a string.
 # Only sets EXPIRE on first call (when prior value was nil).
@@ -216,9 +226,11 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
         """Read current accumulated spend for a session."""
         if self.internal_usage_cache.dual_cache.redis_cache is not None:
             try:
-                result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key)
-                if result is not None:
-                    return float(result)
+                redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
+                    await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key)
+                )
+                if redis_result is not None:
+                    return float(redis_result)
                 return 0.0
             except Exception as e:
                 log_redis_failure(
@@ -228,24 +240,28 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
                     e,
                 )
 
-        result = await self.internal_usage_cache.async_get_cache(
-            key=cache_key,
-            litellm_parent_otel_span=None,
-            local_only=True,
+        cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
+            await self.internal_usage_cache.async_get_cache(
+                key=cache_key,
+                litellm_parent_otel_span=None,
+                local_only=True,
+            )
         )
-        if result is not None:
-            return float(result)
+        if cache_result is not None:
+            return float(cache_result)
         return 0.0
 
     async def _increment_spend(self, cache_key: str, amount: float) -> float:
         """Atomically increment the session spend and return the new value."""
         if self.increment_script is not None:
             try:
-                result: Final = await self.increment_script(
-                    keys=[cache_key],
-                    args=[str(amount), self.ttl],
+                script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python(
+                    await self.increment_script(
+                        keys=[cache_key],
+                        args=[str(amount), self.ttl],
+                    )
                 )
-                return float(result)
+                return float(script_result)
             except Exception as e:
                 log_redis_failure(
                     verbose_proxy_logger,
@@ -257,12 +273,15 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
         return await self._in_memory_increment_spend(cache_key, amount)
 
     async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float:
-        current: Final = await self.internal_usage_cache.async_get_cache(
-            key=cache_key,
-            litellm_parent_otel_span=None,
-            local_only=True,
+        current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
+            await self.internal_usage_cache.async_get_cache(
+                key=cache_key,
+                litellm_parent_otel_span=None,
+                local_only=True,
+            )
         )
-        new_value: Final = (float(current) if current is not None else 0.0) + amount
+        current_spend: Final = float(current) if current is not None else 0.0
+        new_value: Final = current_spend + amount
         await self.internal_usage_cache.async_set_cache(
             key=cache_key,
             value=new_value,

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -42,7 +42,11 @@ from litellm.caching.redis_batch import (
     active_request_redis_batch,
 )
 from litellm.caching.redis_cache import log_redis_failure
-from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
+from litellm.constants import (
+    DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
+)
 from litellm.cost_calculator import get_usage_object
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
@@ -508,11 +512,6 @@ REDIS_NODE_HASHTAG_NAME: Final = "all_keys"
 # *some* output budget; these define that fallback estimate.
 DEFAULT_MAX_TOKENS_ESTIMATE: Final = 4096
 DEFAULT_CHARS_PER_TOKEN: Final = 4
-# Fraction of the available output budget reserved as the upfront floor when
-# the request omits max_tokens. Applied to both DEFAULT_MAX_TOKENS_ESTIMATE
-# (baseline floor) and to the smallest configured TPM limit (capped floor for
-# small per-tenant TPM caps).
-_TPM_FLOOR_FRACTION: Final = 4
 # Both embeddings and the Responses API put their prompt in data["input"],
 # but only embeddings have no output tokens. Every "is this an embedding"
 # check on data["input"] must exclude these call types, or a Responses call
@@ -1003,10 +1002,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         per-tenant cap can't be tripped by the floor alone. Returns the
         baseline floor when no limit is provided.
         """
-        baseline: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
+        baseline: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)
         if min_configured_tpm_limit is None:
             return baseline
-        return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION))
+        return min(baseline, max(1, int(min_configured_tpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)))
 
     @staticmethod
     def _is_embedding_request(data: object, call_type: str | None) -> bool:
@@ -1138,7 +1137,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             if call_type in RESPONSES_API_CALL_TYPES
             else base_capped_floor
         )
-        baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
+        baseline_floor: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)
         is_embedding: Final = PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type)
         if (
             capped_floor >= baseline_floor

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -22,7 +22,7 @@ from typing import Any, Final, Literal, Protocol, cast, overload
 
 import fastapi
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -124,6 +124,20 @@ _USER_MODEL_BUDGET_ADAPTER: Final = TypeAdapter(dict[str, float | BudgetConfig])
 _USER_CACHE_INVALIDATION_BATCH_SIZE: Final = 50
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _AdminTeamAndKeyRows(TypedDict):
+    keys: ReadOnly[Sequence[object] | None]
+    teams: ReadOnly[Sequence[object] | None]
+
+
+_ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter(
+    list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True)
+)
+_ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter(
+    dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
+
 def _user_table(
     prisma_client: "PrismaClient | None",
 ) -> "TableActions[prisma_models.LiteLLM_UserTable]":
@@ -931,6 +945,14 @@ async def user_info(
     request: Request,
     user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"),
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    admin_mcp: bool = fastapi.Query(
+        default=False, description="Include the caller's current admin MCP route permissions"
+    ),
+    mcp_tool: str | None = fastapi.Query(default=None, description="Admin MCP tool to authorize for this caller"),
+    mcp_path: str | None = fastapi.Query(
+        default=None,
+        description="URL-encoded concrete management path used by the MCP tool, before query-string encoding",
+    ),
 ):
     """
     [10/07/2024]
@@ -951,12 +973,23 @@ async def user_info(
         user_id = _normalize_user_info_user_id(request=request, user_id=user_id)
         _enforce_user_info_access(user_id=user_id, user_api_key_dict=user_api_key_dict)
 
+        from litellm.proxy.admin_mcp import get_admin_mcp_permissions
+
+        mcp_permissions: Final = (
+            get_admin_mcp_permissions(user_api_key_dict, request, mcp_tool, mcp_path) if admin_mcp is True else None
+        )
+
         if prisma_client is None:
             raise Exception(
                 "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
             )
         if user_id is None and user_api_key_has_admin_view(user_api_key_dict):
-            return await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict)
+            admin_response: Final = await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict)
+            return (
+                admin_response.model_copy(update={"admin_mcp_permissions": mcp_permissions})
+                if mcp_permissions is not None
+                else admin_response
+            )
         elif user_id is None:
             user_id = user_api_key_dict.user_id
         ## GET USER ROW ##
@@ -995,7 +1028,11 @@ async def user_info(
             ),
         )
 
-        return response_data
+        return (
+            response_data.model_copy(update={"admin_mcp_permissions": mcp_permissions})
+            if mcp_permissions is not None
+            else response_data
+        )
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info(): Exception occured - %s", e)
         raise handle_exception_on_proxy(e)
@@ -1170,8 +1207,8 @@ async def user_info_v2(
 
 async def _fetch_admin_teams_and_keys_rows(
     prisma_client: "PrismaClient", sql_query: str
-) -> Sequence[Mapping[str, Sequence[Mapping[str, object]] | None]]:
-    return await prisma_client.db.query_raw(sql_query)
+) -> Sequence[_AdminTeamAndKeyRows]:
+    return _ADMIN_TEAM_AND_KEY_ROWS_ADAPTER.validate_python(await prisma_client.db.query_raw(sql_query))
 
 
 async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth):
@@ -1201,17 +1238,17 @@ async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth):
 
     verbose_proxy_logger.debug("results_keys: %s", results)
 
-    _keys_in_db: Final[Sequence[Mapping[str, object]]] = results[0]["keys"] or []
+    _keys_in_db: Final[Sequence[object]] = results[0]["keys"] or []
     # cast all keys to LiteLLM_VerificationToken
     keys_in_db: Final = []
     for key in _keys_in_db:
-        key_payload = dict[str, object](key)
+        key_payload = _ADMIN_KEY_ROW_ADAPTER.validate_python(key)
         if key_payload.get("models") is None:
             key_payload["models"] = []
         keys_in_db.append(LiteLLM_VerificationToken.model_validate(key_payload))
 
     # cast all teams to LiteLLM_TeamTable
-    _teams_rows: Final[Sequence[Mapping[str, object]]] = results[0]["teams"] or []
+    _teams_rows: Final[Sequence[object]] = results[0]["teams"] or []
     _teams_in_db: Final = sorted(
         (LiteLLM_TeamTable.model_validate(team) for team in _teams_rows),
         key=lambda x: getattr(x, "team_alias", "") or "",

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -931,6 +931,34 @@
     ],
     "default_model_placeholder": "gpt-3.5-turbo"
   },
+  {
+    "provider": "CLF_AI_GATEWAY",
+    "provider_display_name": "CLF AI Gateway",
+    "litellm_provider": "clf_ai_gateway",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://api.clfaigateway.dev/v1",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "clf_ai_gateway/glm-5.3"
+  },
   {
     "provider": "CLOUDFLARE",
     "provider_display_name": "Cloudflare",

litellm/proxy/realtime_endpoints/endpoints.py

@@ -3,11 +3,13 @@
 import json
 import time
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
 
 import httpx
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi import status as http_status
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
@@ -44,6 +46,20 @@ _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper"
 _ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription")
 
 
+@with_config(ConfigDict(strict=True, extra="allow"))
+class _RealtimeTokenPayload(TypedDict, total=False):
+    v: NotRequired[ReadOnly[object]]
+    ephemeral_key: NotRequired[ReadOnly[str]]
+    model_id: NotRequired[ReadOnly[str]]
+    user_id: NotRequired[ReadOnly[str | None]]
+    team_id: NotRequired[ReadOnly[str | None]]
+    expires_at: NotRequired[ReadOnly[object]]
+    session_type: NotRequired[ReadOnly[str | None]]
+
+
+_REALTIME_TOKEN_PAYLOAD_ADAPTER: Final = TypeAdapter(_RealtimeTokenPayload)
+
+
 def _coerce_realtime_session_type(session_type: str | None) -> str:
     if session_type in _ALLOWED_SESSION_TYPES:
         return session_type
@@ -201,17 +217,14 @@ def _header_value(headers: Mapping[str, str], name: str, default: str) -> str:
 
 def _decode_realtime_token_payload(
     decrypted_value: str,
-) -> dict[str, Any] | None:
+) -> _RealtimeTokenPayload | None:
     """
     Decode realtime token payload; returns None for legacy/raw ephemeral tokens.
     """
     try:
-        decoded: Final = json.loads(decrypted_value)
+        decoded: Final = _REALTIME_TOKEN_PAYLOAD_ADAPTER.validate_python(json.loads(decrypted_value))
     except Exception:
         return None
-
-    if not isinstance(decoded, dict):
-        return None
     if decoded.get("v") != _REALTIME_TOKEN_VERSION:
         return None
     if not isinstance(decoded.get("ephemeral_key"), str):

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -87,6 +87,7 @@ from litellm.types.utils import (
     AzureSpillover,
     CallTypes,
     CostBreakdown,
+    LiteLLMBatch,
     LlmProviders,
     StandardLoggingGuardrailInformation,
     StandardLoggingKubernetesPodRouting,
@@ -96,6 +97,7 @@ from litellm.types.utils import (
     StandardLoggingPayloadErrorInformation,
     StandardLoggingVectorStoreRequest,
     VectorStoreSearchResponse,
+    batch_chat_usage,
 )
 from litellm.utils import get_end_user_id_for_cost_tracking
 
@@ -626,7 +628,12 @@ def get_logging_payload(
         usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict)
     elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict):
         # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models
-        _usage: Final = response_obj_dict.get("usage", None) or {}
+        chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None
+        _usage: Final = (
+            chat_usage_of_batch.model_dump()
+            if chat_usage_of_batch is not None
+            else response_obj_dict.get("usage", None) or {}
+        )
         if isinstance(_usage, litellm.Usage):
             usage = dict(_usage)
         elif isinstance(_usage, dict):

litellm/responses/mcp/litellm_proxy_mcp_handler.py

@@ -35,10 +35,7 @@ from litellm.types.llms.openai import ToolParam as ResponsesToolParam
 from litellm.types.mcp_server.mcp_server_manager import MCPServer
 from litellm.types.utils import (
     CallTypes,
-    ChatCompletionMessageCustomToolCall,
-    ChatCompletionMessageToolCall,
     Choices,
-    Message,
     ModelResponse,
     StandardLoggingMCPToolCall,
 )
@@ -587,29 +584,18 @@ class LiteLLM_Proxy_MCP_Handler:
 
     _extract_tool_calls_from_response = extract_tool_calls_from_response
 
+    @staticmethod
+    def _continued_choice(response: ModelResponse) -> Choices | None:
+        choices: Final = tuple(choice for choice in response.choices if isinstance(choice, Choices))
+        return next((choice for choice in choices if choice.message.tool_calls), next(iter(choices), None))
+
     @staticmethod
     def extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]:
-        """Extract tool calls from a chat completion response."""
-        tool_calls: Final[list[object]] = []
-
-        try:
-            for choice in response.choices:
-                message: Message | None = getattr(choice, "message", None)
-                if message is None:
-                    continue
-                tool_call_entries: (
-                    Sequence[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None
-                ) = getattr(message, "tool_calls", None)
-                if tool_call_entries:
-                    for tool_call in tool_call_entries:
-                        if hasattr(tool_call, "model_dump"):
-                            tool_calls.append(tool_call.model_dump())
-                        else:
-                            tool_calls.append(tool_call)
-        except Exception:
-            verbose_logger.exception("Failed to extract tool calls from chat completion response")
-
-        return tool_calls
+        """Extract the tool calls of the first choice that made any, since the follow-up turn continues it alone."""
+        continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response)
+        if continued_choice is None:
+            return []
+        return [tool_call.model_dump() for tool_call in continued_choice.message.tool_calls or ()]
 
     _extract_tool_calls_from_chat_response = extract_tool_calls_from_chat_response
 
@@ -1018,9 +1004,9 @@ class LiteLLM_Proxy_MCP_Handler:
 
         message_to_append: dict | None = None
         try:
-            first_choice: Final = response.choices[0]
-            if isinstance(first_choice, Choices) and getattr(first_choice, "message", None):
-                message_to_append = first_choice.message.model_dump(exclude_none=True)
+            continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response)
+            if continued_choice is not None:
+                message_to_append = continued_choice.message.model_dump(exclude_none=True)
                 # Ensure tool_calls have arguments field (required by OpenAI API)
                 if message_to_append.get("tool_calls"):
                     for tool_call in message_to_append["tool_calls"]:

litellm/responses/streaming_iterator.py

@@ -43,6 +43,7 @@ from litellm.litellm_core_utils.core_helpers import (
     process_response_headers,
     provider_reported_usage_cost,
     set_response_cost_in_hidden_params,
+    stamp_litellm_bill_on_usage,
 )
 from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
 from litellm.litellm_core_utils.internal_call_metadata import (
@@ -1869,7 +1870,7 @@ def stamp_responses_usage_cost(
     if isinstance(bill, bool) or not isinstance(bill, (int, float)):
         return
     if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)):
-        setattr(usage_obj, "cost", float(bill))
+        stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill))
 
 
 def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:

litellm/router.py

@@ -377,6 +377,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor
     EncryptedContentAffinityCheck,
 )
 from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
+    async_io_token_refund_failure,
     build_io_token_rate_limit_headers,
     deployment_has_io_token_limits,
     refund_stale_reservation_before_retry,
@@ -9500,7 +9501,7 @@ class Router(DeprecatedRouterMembers):
 
             raise original_exception
 
-    async def make_call(self, original_function: Any, *args, **kwargs):
+    async def make_call(self, original_function: Any, *args: object, **kwargs: object):
         """
         Handler for making a call to the .completion()/.embeddings()/etc. functions.
         """
@@ -9509,17 +9510,26 @@ class Router(DeprecatedRouterMembers):
         endpoint_function: Final = (
             generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function)
         )
-        with endpoint_selection(
-            getattr(endpoint_function, "__name__", None),
-            getattr(endpoint_function, "__module__", None),
-            model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group),
-        ):
-            response = original_function(*args, **kwargs)
-            if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
-                response = await response
+        try:
+            with endpoint_selection(
+                getattr(endpoint_function, "__name__", None),
+                getattr(endpoint_function, "__module__", None),
+                model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group),
+            ):
+                response = original_function(*args, **kwargs)
+                if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
+                    response = await response
+        except Exception:
+            with contextlib.suppress(Exception):
+                await async_io_token_refund_failure(
+                    self.cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs)
+                )
+            raise
         await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs)
         ## PROCESS RESPONSE HEADERS
-        response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs)
+        response = await self.set_response_headers(
+            response=response, model_group=model_group if isinstance(model_group, str) else None, request_kwargs=kwargs
+        )
 
         return response
 

litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py

@@ -1,7 +1,8 @@
 """
 Separate ITPM/OTPM (input/output tokens per minute) deployment rate limits.
 
-- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM
+- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM; a request
+  without max_tokens reserves the model's max output, capped at a fraction of the OTPM limit
 - Post-call: reconcile ITPM to actual input tokens and OTPM to actual output tokens
 - Cached prompt-read tokens are excluded from ITPM post-call accounting
 
@@ -22,6 +23,7 @@ from litellm import token_counter
 from litellm._internal_context import with_service_target
 from litellm._logging import verbose_router_logger
 from litellm.caching.dual_cache import DualCache
+from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION
 from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count
 from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
@@ -127,10 +129,15 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] |
     return itpm_key, otpm_key
 
 
-def _tokenizer_model(deployment: object) -> str:
+def _served_model(deployment: object) -> str:
     params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None
     served: Final = params.get("model") if is_str_keyed_mapping(params) else None
-    return deployment_openai_model(deployment) or (served if isinstance(served, str) else "")
+    return served if isinstance(served, str) else ""
+
+
+def _tokenizer_and_served_models(deployment: object) -> tuple[str, str]:
+    served: Final = _served_model(deployment)
+    return deployment_openai_model(deployment) or served, served
 
 
 def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int:
@@ -160,24 +167,61 @@ def _model_max_output_tokens(model_name: str) -> int | None:
     return None
 
 
-def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict) -> int:
-    if request_kwargs:
-        # An explicit max_tokens=0 must be honored, not treated as absent and
-        # replaced by the model default.
-        explicit = request_kwargs.get("max_tokens")
-        if explicit is None:
-            explicit = request_kwargs.get("max_completion_tokens")
-        if explicit is None:
-            explicit = request_kwargs.get("max_output_tokens")
-        if explicit is not None:
-            return max(0, int(explicit))
+_CHAT_OUTPUT_CAP_KEYS: Final = ("max_tokens", "max_completion_tokens")
+_RESPONSES_OUTPUT_CAP_KEYS: Final = ("max_output_tokens",)
+_OUTPUT_CAP_KEYS: Final = _CHAT_OUTPUT_CAP_KEYS + _RESPONSES_OUTPUT_CAP_KEYS
 
-    model_name: Final = (deployment.get("litellm_params") or {}).get("model")
-    if model_name:
-        model_max: Final = _model_max_output_tokens(model_name)
-        if model_max is not None:
-            return model_max
-    return 4096
+
+def _explicit_max_tokens(
+    request_kwargs: Mapping[str, object] | None, cap_keys: tuple[str, ...] = _OUTPUT_CAP_KEYS
+) -> int | None:
+    if not request_kwargs:
+        return None
+    explicit: Final = next(
+        (request_kwargs[key] for key in cap_keys if request_kwargs.get(key) is not None),
+        None,
+    )
+    match explicit:
+        case int() | float() | str():
+            return max(0, int(explicit))
+        case _:
+            return None
+
+
+def _sent_deployment_cap_keys(request_kwargs: Mapping[str, object] | None) -> tuple[str, ...]:
+    is_responses_request: Final = (
+        request_kwargs is not None and "input" in request_kwargs and "messages" not in request_kwargs
+    )
+    return _RESPONSES_OUTPUT_CAP_KEYS if is_responses_request else _CHAT_OUTPUT_CAP_KEYS
+
+
+def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployment: Mapping[str, object]) -> int | None:
+    params: Final = deployment.get("litellm_params")
+    return (
+        _explicit_max_tokens(params, _sent_deployment_cap_keys(request_kwargs))
+        if is_str_keyed_mapping(params)
+        else None
+    )
+
+
+def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int:
+    model_max: Final = _model_max_output_tokens(model_name) if model_name else None
+    share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))
+    return min(4096 if model_max is None else model_max, share_of_limit)
+
+
+def _otpm_reservation(
+    request_kwargs: Mapping[str, object] | None,
+    deployment: Mapping[str, object],
+    model_name: str,
+    otpm_limit: int,
+) -> int:
+    from_request: Final = _explicit_max_tokens(request_kwargs)
+    if from_request is not None:
+        return from_request
+    default: Final = _default_otpm_reservation(model_name, otpm_limit)
+    from_deployment: Final = _deployment_output_cap(request_kwargs, deployment)
+    return default if from_deployment is None else min(from_deployment, default)
 
 
 def _get_usage_tokens(usage: object) -> tuple[int, int, int]:
@@ -210,6 +254,12 @@ def _extract_response_usage(response_obj: object) -> object:
     return getattr(response_obj, "usage", None)
 
 
+def _openai_shaped_usage(usage: object) -> object:
+    if isinstance(usage, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage):
+        return litellm.AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None)
+    return usage
+
+
 def _usage_is_present(usage: object) -> bool:
     """
     True only if usage carries an actual input/output breakdown.
@@ -238,7 +288,7 @@ def _resolve_reconcile_usage_tokens(
     fields. When usage cannot be resolved, return ``usage_resolved=False`` so
     callers keep the pre-call reservation instead of refunding it as zero usage.
     """
-    usage: Final = _extract_response_usage(response_obj)
+    usage: Final = _openai_shaped_usage(_extract_response_usage(response_obj))
     if _usage_is_present(usage):
         prompt_tokens, completion_tokens, cached_tokens = _get_usage_tokens(usage)
         return max(0, prompt_tokens - cached_tokens), completion_tokens, True
@@ -261,6 +311,21 @@ def _resolve_reconcile_usage_tokens(
     return 0, 0, False
 
 
+def _key_for_this_minute(reserved_key: str) -> str:
+    counter, _, _ = reserved_key.rpartition(":")
+    return f"{counter}:{get_utc_datetime().strftime('%H-%M')}"
+
+
+def _reconcile_increments(
+    reservation: tuple[int, int, str | None, str | None], billable_input: int, completion_tokens: int
+) -> tuple[tuple[str, int], ...]:
+    itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation
+    deltas: Final = ((itpm_key, billable_input - itpm_reserved), (otpm_key, completion_tokens - otpm_reserved))
+    return tuple(
+        (key if delta < 0 else _key_for_this_minute(key), delta) for key, delta in deltas if key is not None and delta
+    )
+
+
 def _stash_reservation_in_metadata(
     request_kwargs: dict[str, object] | None,
     *,
@@ -311,7 +376,9 @@ def _reservation_channels(kwargs: Mapping[str, object] | None) -> tuple[object,
         return ()
     top_level: Final = (kwargs.get("metadata"), kwargs.get("litellm_metadata"))
     litellm_params: Final = _as_mutable_mapping(kwargs.get("litellm_params"))
-    from_params: Final = () if litellm_params is None else (litellm_params.get("metadata"),)
+    from_params: Final = (
+        () if litellm_params is None else (litellm_params.get("metadata"), litellm_params.get("litellm_metadata"))
+    )
     standard_logging_object: Final = _as_mutable_mapping(kwargs.get("standard_logging_object"))
     from_logging_object: Final = () if standard_logging_object is None else (standard_logging_object.get("metadata"),)
     return top_level + from_params + from_logging_object
@@ -433,9 +500,11 @@ def io_token_pre_call_check(
         return deployment
 
     request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
-    _model: Final = _tokenizer_model(deployment)
+    _model, served_model = _tokenizer_and_served_models(deployment)
     estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
-    max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
+    otpm_to_reserve: Final = (
+        _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
+    )
 
     dt: Final = get_utc_datetime()
     current_minute: Final = dt.strftime("%H-%M")
@@ -458,7 +527,7 @@ def io_token_pre_call_check(
         )
 
     if otpm_limit is not None:
-        otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit)
+        otpm_reserved = otpm_to_reserve
         try:
             _sync_increment_with_rollback(
                 dual_cache,
@@ -497,9 +566,11 @@ async def async_io_token_pre_call_check(
         return deployment
 
     request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
-    _model: Final = _tokenizer_model(deployment)
+    _model, served_model = _tokenizer_and_served_models(deployment)
     estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
-    max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
+    otpm_to_reserve: Final = (
+        _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
+    )
 
     dt: Final = get_utc_datetime()
     current_minute: Final = dt.strftime("%H-%M")
@@ -523,7 +594,7 @@ async def async_io_token_pre_call_check(
         )
 
     if otpm_limit is not None:
-        otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit)
+        otpm_reserved = otpm_to_reserve
         try:
             await _increment_with_rollback(
                 dual_cache,
@@ -564,7 +635,8 @@ def io_token_reconcile_success(
 ) -> None:
     request_kwargs: Final[Mapping[str, object] | None] = kwargs
     response: Final[object] = response_obj
-    itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs)
+    reservation: Final = _read_reservation_from_kwargs(request_kwargs)
+    itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation
     if itpm_key is None and otpm_key is None:
         return
 
@@ -572,23 +644,8 @@ def io_token_reconcile_success(
 
     try:
         if usage_resolved:
-            if itpm_key is not None:
-                itpm_delta: Final = billable_input - itpm_reserved
-                if itpm_delta != 0:
-                    dual_cache.increment_cache(
-                        key=itpm_key,
-                        value=itpm_delta,
-                        ttl=RoutingArgsTTL,
-                    )
-
-            if otpm_key is not None:
-                otpm_delta: Final = completion_tokens - otpm_reserved
-                if otpm_delta != 0:
-                    dual_cache.increment_cache(
-                        key=otpm_key,
-                        value=otpm_delta,
-                        ttl=RoutingArgsTTL,
-                    )
+            for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens):
+                dual_cache.increment_cache(key=key, value=delta, ttl=RoutingArgsTTL)
         else:
             verbose_router_logger.debug(
                 "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)",
@@ -618,39 +675,23 @@ async def async_io_token_reconcile_success(
 ) -> None:
     request_kwargs: Final[Mapping[str, object] | None] = kwargs
     response: Final[object] = response_obj
-    itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs)
+    reservation: Final = _read_reservation_from_kwargs(request_kwargs)
+    itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation
     if itpm_key is None and otpm_key is None:
         return
 
     billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response)
 
-    # Reconcile against the exact key that held the reservation (which encodes
-    # the reservation's minute), not a key recomputed at response time. This
-    # tracks actual usage even when the pre-call estimate was 0, and avoids a
-    # minute-boundary mismatch for calls that span into the next minute. Always
-    # clear the stash afterwards (even if an increment throws) so a retry or a
-    # duplicate success event can't re-process it.
+    # A refund goes back to the minute that held the reservation, while usage above the reservation is charged
+    # to the minute the response finished in, so a call that runs past its minute still counts against a live
+    # window. Always clear the stash afterwards (even if an increment throws) so a retry or a duplicate success
+    # event can't re-process it.
     try:
         if usage_resolved:
-            if itpm_key is not None:
-                itpm_delta: Final = billable_input - itpm_reserved
-                if itpm_delta != 0:
-                    await dual_cache.async_increment_cache(
-                        key=itpm_key,
-                        value=itpm_delta,
-                        ttl=RoutingArgsTTL,
-                        parent_otel_span=parent_otel_span,
-                    )
-
-            if otpm_key is not None:
-                otpm_delta: Final = completion_tokens - otpm_reserved
-                if otpm_delta != 0:
-                    await dual_cache.async_increment_cache(
-                        key=otpm_key,
-                        value=otpm_delta,
-                        ttl=RoutingArgsTTL,
-                        parent_otel_span=parent_otel_span,
-                    )
+            for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens):
+                await dual_cache.async_increment_cache(
+                    key=key, value=delta, ttl=RoutingArgsTTL, parent_otel_span=parent_otel_span
+                )
         else:
             verbose_router_logger.debug(
                 "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)",

litellm/router_utils/pre_call_checks/model_rate_limit_check.py

@@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache
 from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
-    ITPM_RESERVED_KEY,
     async_io_token_pre_call_check,
     async_io_token_reconcile_success,
     async_io_token_refund_failure,
@@ -485,24 +484,12 @@ class ModelRateLimitingCheck(CustomLogger):
 
         try:
             standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object")
-
-            # IO token reconciliation works purely from the cache keys stashed in
-            # kwargs/metadata, so it must run before the model_id guard below
-            # (which only the TPM-tracking path needs). Otherwise a request whose
-            # standard_logging_object lacks model_id would never return its
-            # reservation, leaving the counter elevated until the TTL expires.
-            slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {}
-            kwargs_metadata: Final = kwargs.get("metadata") or {}
-            if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata:
-                await async_io_token_reconcile_success(
-                    self.dual_cache,
-                    kwargs,
-                    response_obj,
-                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
-                )
-                # Fall through: a deployment can also configure tpm/rpm alongside
-                # itpm/otpm, and that path's pre-call check reads the tpm_key
-                # counter tracked below, so it must still be incremented here.
+            await async_io_token_reconcile_success(
+                self.dual_cache,
+                kwargs,
+                response_obj,
+                parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
+            )
 
             if standard_logging_object is None:
                 return
@@ -557,17 +544,7 @@ class ModelRateLimitingCheck(CustomLogger):
         """
         try:
             standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object")
-            slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {}
-            kwargs_metadata: Final = kwargs.get("metadata") or {}
-            if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata:
-                io_token_reconcile_success(
-                    self.dual_cache,
-                    kwargs,
-                    response_obj,
-                )
-                # Fall through: a deployment can also configure tpm/rpm alongside
-                # itpm/otpm, and that path's pre-call check reads the tpm_key
-                # counter tracked below, so it must still be incremented here.
+            io_token_reconcile_success(self.dual_cache, kwargs, response_obj)
 
             if standard_logging_object is None:
                 return

litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py

@@ -2,9 +2,10 @@
 
 import enum
 from collections.abc import Mapping
-from typing import Any, Literal, TypedDict
+from typing import Literal, TypedDict
 
 from pydantic import Field
+from typing_extensions import ReadOnly
 
 from .base import GuardrailConfigModel
 
@@ -42,7 +43,7 @@ class PolicyApplicableTo(str, enum.Enum):
 class DynamoAIRequest(TypedDict, total=False):
     """Request structure for DynamoAI /moderation/analyze endpoint"""
 
-    messages: list[dict[str, Any]]
+    messages: ReadOnly[list[dict[str, object]]]
     textType: DynamoTextType | None
     policyIds: list[str]
     modelId: str | None
@@ -58,8 +59,8 @@ class PolicyInfo(TypedDict, total=False):
     description: str
     method: PolicyMethod
     action: Literal["BLOCK", "WARN", "REDACT", "SANITIZE", "NONE"]
-    methodParams: dict[str, Any]
-    decisionParams: dict[str, Any]
+    methodParams: ReadOnly[dict[str, object]]
+    decisionParams: ReadOnly[dict[str, object]]
     applicableTo: PolicyApplicableTo
     created_at: str
     creatorId: str
@@ -76,7 +77,7 @@ class AppliedPolicyDto(TypedDict, total=False):
     """Applied policy details from DynamoAI response"""
 
     policy: PolicyInfo
-    outputs: dict[str, Any] | None
+    outputs: ReadOnly[dict[str, object] | None]
     action: str | None