Merge remote-tracking branch 'origin/main' into litellm_aiand_brand_drive

# Conflicts:
#	litellm/litellm_core_utils/llm_request_utils.py
This commit is contained in:
mateo-berri 2026-10-30 04:05:03 -07:00
commit 95ad02f344
317 changed files with 19122 additions and 5247 deletions

View file

@ -8,6 +8,7 @@ on:
permissions:
contents: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@ -21,16 +22,19 @@ jobs:
with:
ref: ${{ github.event.pull_request.base.sha }}
persist-credentials: false
- name: Fetch the pull request head, its merge base, and the base branch tip
- name: Check out the base branch tip and fetch the pull request head and its merge base
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
git checkout -q --detach "$target"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
echo "target=$target" >> "$GITHUB_OUTPUT"
- name: Set up uv

View file

@ -1,152 +0,0 @@
name: Lens installation smoke
on:
workflow_dispatch:
inputs:
lens_image:
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
required: true
type: string
lens_upgrade_image:
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
required: true
type: string
gateway_baseline_ref:
description: Previously qualified gateway source commit, full SHA
required: true
type: string
permissions:
contents: read
concurrency:
group: lens-install-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-images:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- component: gateway
dockerfile: gateway/Dockerfile
- component: backend
dockerfile: backend/Dockerfile
- component: ui
dockerfile: ui/Dockerfile
- component: migrations
dockerfile: migrations/Dockerfile
- component: monolith
dockerfile: Dockerfile
- component: gateway
dockerfile: gateway/Dockerfile
baseline: true
- component: backend
dockerfile: backend/Dockerfile
baseline: true
- component: monolith
dockerfile: Dockerfile
baseline: true
env:
COMPONENT: ${{ matrix.component }}
DOCKERFILE: ${{ matrix.dockerfile }}
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
CURRENT_REF: ${{ github.sha }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ env.SOURCE_REF }}
- name: Verify the selected source
shell: bash -euo pipefail {0}
run: |
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
test "$SOURCE_REF" != "$CURRENT_REF"
fi
- name: Build the matching release image
run: |
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
- name: Save the matching release image
run: |
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
compression-level: 0
retention-days: 3
if-no-files-found: error
overwrite: true
helm-install:
needs: build-images
runs-on: ubuntu-latest-16-cores
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-install-*-${{ github.sha }}
merge-multiple: true
path: ${{ runner.temp }}/lens-install-images
- name: Load the matching release images
run: |
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
gzip -dc "$archive" | docker load
rm "$archive"
done
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
with:
cosign-release: v2.4.3
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ vars.LENS_READ_USER || github.actor }}
password: ${{ secrets.LENS_READ_TOKEN }}
- name: Verify independent Lens images before loading them
env:
LENS_IMAGE: ${{ inputs.lens_image }}
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
shell: bash -euo pipefail {0}
run: |
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
docker pull "$image"
done
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
- name: Install pinned Kubernetes test tools
run: |
curl --fail --location --output "$RUNNER_TEMP/kind" \
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
chmod +x "$RUNNER_TEMP/kind"
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
chmod +x "$RUNNER_TEMP/kubectl"
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
- name: Install, ingest, upgrade, and restart both charts
run: bash tests/e2e/migrations/lens_helm_smoke.sh

View file

@ -19,17 +19,17 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.base.sha }}
ref: ${{ github.event.pull_request.base.ref }}
persist-credentials: false
- name: Fetch the pull request head, its merge base and its reviews
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

View file

@ -48,11 +48,13 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$MERGE_BASE"
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@ -264,6 +266,7 @@ jobs:
name: ui
permissions:
contents: read
pull-requests: read
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
@ -281,16 +284,12 @@ jobs:
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# base.sha is the base branch tip from when the PR was opened, while
# actions/checkout leaves HEAD on a merge of the PR into the *current*
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
# landed since, so a PR that touches no UI file still gets linted
# against hundreds of other people's files. Diff the PR head against its
# own merge base instead, which is exactly what this PR changed.
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
@ -630,6 +629,12 @@ jobs:
- name: check_migrations_lock_timeout
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
- name: check_no_none_string_error_fields
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
- name: test_check_no_none_string_error_fields
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py

View file

@ -245,6 +245,7 @@ jobs:
tests/unit/llms/chatgpt
tests/unit/llms/clarifai
tests/unit/llms/claude_code
tests/unit/llms/clf_ai_gateway
tests/unit/llms/cloudflare
tests/unit/llms/codestral
tests/unit/llms/codex
@ -1150,18 +1151,20 @@ jobs:
env:
CI: "true"
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
if [ -z "$BASE_SHA" ]; then
if [ -z "$PR_NUMBER" ]; then
echo "Push to $GITHUB_REF_NAME: running the full suite"
full_suite
exit 0
fi
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()

View file

@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -5,8 +5,5 @@ dependencies:
- name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: 18.19.1
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
generated: '2026-10-09T04:18:10.643228000Z'
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
generated: '2026-10-30T02:13:11.505726000Z'

View file

@ -39,6 +39,3 @@ dependencies:
version: "18.19.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -256,9 +256,7 @@ itself.
## Connect Lens
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
## Shutdown accounting

View file

@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -402,95 +398,28 @@ shutdown drain window.
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -1,8 +1,6 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "litellm.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- $lensMode := include "litellm.lens.mode" . -}}
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@ -46,18 +44,18 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
name: {{ $.Values.lens.service.name }}
port:
number: {{ $.Values.lensWorker.service.port }}
number: {{ $.Values.lens.service.port }}
{{- else }}
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
servicePort: {{ $.Values.lensWorker.service.port }}
serviceName: {{ $.Values.lens.service.name }}
servicePort: {{ $.Values.lens.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -0,0 +1,128 @@
suite: lens connection
templates:
- deployment.yaml
- configmap-litellm.yaml
- ingress.yaml
tests:
- it: connects the proxy to an external Lens
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service ahead of the host paths
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- equal:
path: spec.rules[0].http.paths[0].path
value: /lens-ingest
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,84 +0,0 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

View file

@ -1,187 +0,0 @@
---
suite: Lens deployment modes
templates:
- ingress.yaml
- configmap-litellm.yaml
- deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,52 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- configmap-litellm.yaml
- deployment.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,59 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

View file

@ -1,167 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- configmap-litellm.yaml
- deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects deployment.yaml to the shared Lens service
template: deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.hosts:
- host: gateway.example
paths:
- path: /
pathType: Prefix
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference

View file

@ -1,162 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

View file

@ -668,70 +668,15 @@ serviceMonitor:
# - test-namespace
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -1,6 +0,0 @@
dependencies:
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
generated: '2026-10-09T04:18:10.642706000Z'

View file

@ -6,8 +6,3 @@ version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
dependencies:
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -1,10 +1,8 @@
# LiteLLM componentized chart
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
This chart deploys separate gateway, backend and UI services
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.

View file

@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
collector containers through an emptyDir. Empty when the sidecar is off
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
*/}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -635,93 +631,27 @@ shutdown drain window.
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -166,14 +166,14 @@ spec:
- host: {{ $componentHosts.gateway | quote }}
http:
paths:
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
- path: /
pathType: Prefix
@ -249,15 +249,15 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -15,7 +15,6 @@ tests:
ingress.componentHosts.ui: ui.example.com
ingress.componentHosts.backend: backend.example.com
ingress.componentHosts.gateway: gateway.example.com
lensWorker.mode: disabled
ingress.tls:
- hosts:
- ui.example.com
@ -83,12 +82,12 @@ tests:
ui.backendUrl: https://backend.example.com
ui.publicUrl: https://ui.example.com
backend.gatewayUrl: https://gateway.example.com
lensWorker.mode: external
lensWorker.externalServiceName: existing-lens
lensWorker.externalUrl: https://lens.example.com
lensWorker.publicUrl: https://lens.example.com
lensWorker.serviceTokenSecret.name: existing-lens-signing
lensWorker.gateway.secretName: existing-lens-signing
lens.enabled: true
lens.service.name: existing-lens
lens.url: http://existing-lens:4318
lens.publicUrl: https://gateway.example.com/lens-ingest
lens.serviceTokenSecret.name: existing-lens-signing
lens.gatewaySecret.name: existing-lens-signing
asserts:
- equal:
path: spec.rules[2].http.paths[0].path

View file

@ -0,0 +1,166 @@
suite: lens connection
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
- gateway/configmap.yaml
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: connects the gateway to an external Lens
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: connects the backend to an external Lens
template: backend/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service on the single ingress host
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,86 +0,0 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

View file

@ -1,208 +0,0 @@
---
suite: Lens deployment modes
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
- ingress.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: keeps the ingestion endpoint on an independently deployed Lens service
template: ingress.yaml
set:
ingress.enabled: true
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: existing-lens
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,54 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: backend/deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,61 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

View file

@ -1,196 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- gateway/configmap.yaml
- gateway/deployment.yaml
- backend/deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects gateway/deployment.yaml to the shared Lens service
template: gateway/deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: connects backend/deployment.yaml to the shared Lens service
template: backend/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.host: gateway.example
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: gateway/deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: omits Lens connection settings when disabled in backend/deployment.yaml
template: backend/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: gateway/deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference
values:
- ./values/required.yaml

View file

@ -1,164 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

View file

@ -1,79 +0,0 @@
suite: Independent Lens release and credentials
templates:
- lens/deployment.yaml
values:
- ./values/required.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
tests:
- it: selects a Lens release independently of the gateway
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
lensWorker.image.tag: 2.3.4
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:2.3.4
- it: retains the pinned Lens chart default when only the gateway changes
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:0.1.0-dev.0
- it: selects development publications explicitly
set:
lensWorker.image.repository: ghcr.io/berriai/lens-dev
lensWorker.image.tag: sha-0123456789abcdef
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
- it: preserves a verified registry mirror and digest
set:
lensWorker.image.repository: registry.example/lens
lensWorker.image.tag: replaced-label
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- it: rejects a malformed digest
set:
lensWorker.image.digest: sha256:invalid
asserts:
- failedTemplate:
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
- it: keeps the legacy enablement switch working
set:
lensWorker.enabled: false
asserts:
- hasDocuments: {count: 0}
- it: keeps isolated local analysis with delegated gateway access
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: LENS_MODE, value: standalone}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- equal:
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
value: true
- equal:
path: spec.template.spec.volumes[0].emptyDir
value: {medium: Memory, sizeLimit: 1Gi}

View file

@ -654,70 +654,15 @@ ui:
topologySpreadConstraints: []
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -238,7 +238,8 @@ impl AnthropicBeta {
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
| Self::ThinkingBindingControls20260801
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -275,7 +276,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,

View file

@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
baseten_models: Set[str] = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
@ -849,6 +850,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.add(key)
elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map
clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key
elif value.get("litellm_provider") == "perplexity":
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
@ -859,6 +862,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
baseten_models.discard(key)
@ -1103,6 +1111,7 @@ model_list = list(
| set(ollama_models)
| bedrock_models
| deepinfra_models
| clf_ai_gateway_models
| perplexity_models
| set(maritalk_models)
| runwayml_models
@ -1228,6 +1237,7 @@ def _build_models_by_provider() -> dict:
"ollama": ollama_models,
"ollama_chat": ollama_models,
"deepinfra": deepinfra_models,
"clf_ai_gateway": clf_ai_gateway_models,
"perplexity": perplexity_models,
"maritalk": maritalk_models,
"watsonx": watsonx_models,
@ -2104,6 +2114,9 @@ if TYPE_CHECKING:
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
)
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
from .llms.clf_ai_gateway.chat.transformation import (
ClfAiGatewayConfig as _ClfAiGatewayConfig,
)
from .llms.llamafile.chat.transformation import (
LlamafileChatConfig as _LlamafileChatConfig,
)
@ -2133,6 +2146,7 @@ if TYPE_CHECKING:
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
DeepInfraConfig: Type[_DeepInfraConfig]
ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
LlamafileChatConfig: Type[_LlamafileChatConfig]
LMStudioChatConfig: Type[_LMStudioChatConfig]
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]

View file

@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
"MistralEmbeddingConfig",
"OpenAIImageVariationConfig",
"DeepInfraConfig",
"ClfAiGatewayConfig",
"DeepgramAudioTranscriptionConfig",
"TopazImageVariationConfig",
"OpenAITextCompletionConfig",
@ -975,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenAIImageVariationConfig",
),
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
"ClfAiGatewayConfig": (
".llms.clf_ai_gateway.chat.transformation",
"ClfAiGatewayConfig",
),
"DeepgramAudioTranscriptionConfig": (
".llms.deepgram.audio_transcription.transformation",
"DeepgramAudioTranscriptionConfig",

View file

@ -18,7 +18,7 @@ from contextlib import contextmanager
from enum import Enum
from typing import Any, Final, Literal, cast
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
import litellm
from litellm._internal_context import current_service_target, service_target
@ -43,6 +43,9 @@ from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
@contextmanager
@ -207,7 +210,7 @@ class Cache:
if not redis_startup_nodes:
_env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
redis_startup_nodes = json.loads(_env_cluster_nodes)
redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
if redis_startup_nodes:
# Only pass GCP parameters if they are provided

View file

@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
self._structured_output: Final = structured_output
self._first_message_item_done: bool = False
self._reasoning_summary_streamed: bool = False
def _handle_string_chunk(
self, str_line: Union[str, "BaseModel"]
@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
replays_openai_history: bool = False,
structured_output: bool = False,
reasoning_summary_streamed: bool = False,
) -> "ModelResponseStream":
"""
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
replays_openai_history: Record the completed output's item sequence for the next turn
structured_output: Only the first message item became content
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
Returns:
ModelResponseStream: OpenAI-formatted streaming chunk
@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
else:
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
elif event_type == "response.reasoning_summary_text.delta":
content_part = parsed_chunk.get("delta", None)
if content_part:
return ModelResponseStream(
choices=[
StreamingChoices(
index=cast(int, parsed_chunk.get("summary_index")),
index=0,
delta=Delta(reasoning_content=content_part),
)
]
@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map=self._tool_call_index_map,
replays_openai_history=self._replays_openai_history,
structured_output=self._structured_output,
reasoning_summary_streamed=self._reasoning_summary_streamed,
)
)
if self._structured_output and _is_message_item_done(chunk):
self._first_message_item_done = True
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
self._reasoning_summary_streamed = True
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

View file

@ -7,6 +7,7 @@ from typing import Final, Literal
from litellm.litellm_core_utils.env_utils import (
get_env_float,
get_env_float_in_range,
get_env_float_or_none,
get_env_int,
get_env_int_in_range,
@ -149,6 +150,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
"DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
) # default cooldown a deployment if 50% of requests fail in a given minute
DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
)
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
@ -416,6 +420,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@ -753,7 +759,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
"low": 1,
@ -813,6 +825,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"ollama",
"ollama_chat",
"deepinfra",
"clf_ai_gateway",
"perplexity",
"mistral",
"groq",
@ -1012,6 +1025,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"api.perplexity.ai",
"api.endpoints.anyscale.com/v1",
"api.deepinfra.com/v1/openai",
"api.clfaigateway.dev/v1",
"api.mistral.ai/v1",
"codestral.mistral.ai/v1/chat/completions",
"codestral.mistral.ai/v1/fim/completions",
@ -1083,6 +1097,7 @@ openai_compatible_providers: Final[list[str]] = [
"deepseek",
"tencent",
"deepinfra",
"clf_ai_gateway",
"perplexity",
"xinference",
"xai",
@ -1261,7 +1276,7 @@ empower_models: Final = set(
]
)
together_ai_models: Final[set] = set(
together_ai_models: Final[set[str]] = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",

View file

@ -153,7 +153,9 @@ from litellm.utils import (
ProviderConfigManager,
TextCompletionResponse,
TranscriptionResponse,
cached_cost_map_key,
cached_get_model_info_helper,
mapped_model_fill_missing,
model_info_with_pricing_overrides,
runtime_model_cost_registrations,
token_counter,
@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
"""
1. If custom pricing is true, return received model name
2. If base_model is set (e.g. for azure models), return that
3. If completion response has model set return that
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
4. Check if model is passed in return that
"""
return_model: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
if completion_response is not None:
if isinstance(completion_response, BaseModel):
completion_response_model = getattr(completion_response, "model", None)
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
completion_response_model: Final = _get_response_model(completion_response)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
explicit_pricing: Final = custom_pricing is True or base_model is not None
configured_model: Final = model or (
completion_response_model if isinstance(completion_response_model, str) else None
@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
else:
return_model = model
elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
elif base_model is not None or bedrock_pricing_model is not None:
return_model = base_model if base_model is not None else bedrock_pricing_model
elif provider_response_model is not None:
return _served_or_priced_requested_model(
provider_response_model, requested_model, custom_llm_provider, priced_region
)
elif completion_response_model is None and hidden_params is not None:
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
return_model = hidden_params.get("model", model)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model
return _served_or_priced_requested_model(
completion_response_model, requested_model, custom_llm_provider, priced_region
)
if return_model is None and model is not None:
return_model = model
if (
return_model is not None
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
provider_prefix: Final = (
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
)
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
return return_model
if return_model is None:
return None
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
return model
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
def _served_or_priced_requested_model(
served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
) -> str:
served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
if requested_model is None:
return served
requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
if served in (requested, requested.partition("/")[2]):
return served
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
if _resolves_to_rates(served, pricing_provider):
return served
return requested if _resolves_to_rates(requested, pricing_provider) else served
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@ -1509,6 +1544,12 @@ def _response_time_ms_for_cost(
return litellm_logging_obj.get_response_ms()
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
def _adjusted_provider_billed_cost(
reported_cost: float,
custom_llm_provider: str | None,
@ -1526,9 +1567,9 @@ def _adjusted_provider_billed_cost(
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=None,
completion_tokens_cost_usd_dollar=None,
cost_for_built_in_tools_cost_usd_dollar=None,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=adjusted_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
discount_percent=discount_percent,
@ -2341,6 +2382,15 @@ def response_cost_calculator(
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=provider_response_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=provider_response_cost,
original_cost=provider_response_cost,
)
return provider_response_cost
response_cost = completion_cost(
@ -3516,7 +3566,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
session that should bill nothing fell through to the public rates instead.
"""
resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
entries: Final = (
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
litellm.model_cost.get(model_name),
@ -3712,9 +3762,9 @@ def handle_realtime_transcription_cost_calculation(
return 0.0
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
override_info: Final = (
_get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
)
total_cost = 0.0
@ -3745,13 +3795,6 @@ def _get_transcription_model_name_from_results(
return None
def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
return None
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]

View file

@ -9,6 +9,7 @@
## LiteLLM versions of the OpenAI Exception Types
import datetime
import enum
from collections.abc import Mapping, Sequence
from typing import Any, Final
@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
return None
def _upstream_body(response: object) -> bytes:
if not isinstance(response, httpx.Response):
return b""
try:
return response.content
except httpx.ResponseNotRead:
return b""
def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
if not isinstance(upstream, httpx.Response):
return httpx.Request(method="POST", url=fallback_url)
try:
upstream_request: Final = upstream.request
except RuntimeError:
return httpx.Request(method="POST", url=fallback_url)
return httpx.Request(
method=upstream_request.method,
url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
)
def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
if not isinstance(upstream, httpx.Response):
return None
try:
return upstream.elapsed
except RuntimeError:
return None
def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
response: Final = httpx.Response(
status_code=status_code,
stream=httpx.ByteStream(_upstream_body(upstream)),
request=_request_without_credentials(upstream, fallback_url),
)
_ = response.read()
response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
elapsed: Final = _upstream_elapsed(upstream)
if elapsed is not None:
response.elapsed = elapsed
return response
_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
_MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
# headers stay reachable on `e.response.headers` for callers that
# explicitly want them; only the proxy-supplied `headers=` kwarg
# makes it onto `self.headers`.
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
# Mirrors FastAPI HTTPException.detail so the same instance can be
# serialized through both the ProxyException and HTTPException paths.
self.detail = detail if detail is not None else self.message
self.response = httpx.Response(
status_code=429,
headers=_response_headers,
content=response.content if response is not None else None,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message, response=self.response, body=None
) # Call the base class constructor with the parameters it needs
@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -908,6 +924,7 @@ class APIError(openai.APIError):
max_retries: int | None = None,
num_retries: int | None = None,
body: object | None = None,
response: httpx.Response | None = None,
):
self.status_code = status_code
self.message = f"litellm.APIError: {message}"
@ -916,6 +933,8 @@ class APIError(openai.APIError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
if response is not None:
self.response = response
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=body)

View file

@ -20,7 +20,7 @@ class ImageEditRequestUtils:
image_edit_optional_params: ImageEditOptionalRequestParams,
drop_params: bool | None = None,
additional_drop_params: Collection[str] | None = None,
) -> dict:
) -> dict[str, object]:
"""
Get optional parameters for the image edit API.

View file

@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
with_prompt_cache_breakpoint,
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
def _hosted_openai_dialect_flag(
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
"""
import litellm
exact_entry: Final = litellm.model_cost.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
return None
provider: Final = custom_llm_provider or resolve_provider(model)
if provider is None or provider == "openai":
return None
if isinstance(exact_entry, dict) and exact_provider == provider:
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
from litellm.utils import get_potential_model_names
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)

View file

@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
import urllib.parse
from typing import Final
from pydantic import BaseModel, ConfigDict
from litellm.llms.custom_httpx.http_handler import HTTPHandler
class _PromptVersionResponse(BaseModel):
model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
data: dict[str, object] | None = None
def _sanitize_id(identifier: str) -> str:
"""Reject path traversal characters and URL-encode the identifier."""
if any(c in identifier for c in ("/", "\\", "#", "?")):
@ -71,8 +79,7 @@ class ArizePhoenixClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final = response.json()
return data.get("data")
return _PromptVersionResponse.model_validate(response.json()).data
except Exception as e:
# Check if it's an HTTP error

View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
)
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
if not isinstance(metadata, dict):
return []
return metadata.get("guardrails") or []
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
return True
raise
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
"""
Returns the guardrail(s) to be run from the metadata or root
Returns the guardrail(s) to be run from the root, else from both metadata fields together
"""
if "guardrails" in data:
return data["guardrails"]
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
# writes to "metadata"; for thread/assistant endpoints it writes to
# "litellm_metadata". We check the one that actually contains the "guardrails"
# key so that a non-empty litellm_metadata without guardrails does not shadow
# the merged list stored in metadata (which would cause team guardrails to be
# silently skipped while default_on=True policy guardrails still fire).
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key) or {}
if isinstance(meta, dict) and "guardrails" in meta:
return meta.get("guardrails") or []
return []
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
return [
*_guardrails_listed_in(request_fields.get("metadata")),
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
]
def _guardrail_is_in_requested_guardrails(
self,
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
) -> bool:
for _guardrail in requested_guardrails:
if isinstance(_guardrail, dict):

View file

@ -72,7 +72,7 @@ if TYPE_CHECKING:
)
from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
Span = _Span | Any
Span = _Span
Tracer = _Tracer
Context = _Context
SpanExporter = _SpanExporter
@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
span = None
# Only set attributes if the span is still recording (not closed)
# Note: parent_span is guaranteed to be not None here
if hasattr(parent_span, "set_status"):
if parent_span is not None and hasattr(parent_span, "set_status"):
parent_span.set_status(Status(StatusCode.OK))
self.set_attributes(parent_span, kwargs, response_obj)
# Raw-request as direct child of parent_span
@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# record error on parent span (keeps hierarchy shallow)
# Only set attributes if the span is still recording (not closed)
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span.is_recording():
if parent_otel_span is not None and parent_otel_span.is_recording():
self.set_attributes(parent_otel_span, kwargs, response_obj)
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))

View file

@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
emit_native_blocks: Final = native_tool is not None or (
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
)
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
content: Final[list[dict[str, object]]] = []
if native_tool is not None:
if emit_native_blocks:
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
tool_name: Final = native_tool.get("name") or "web_search"
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
content.append(
{
"type": "server_tool_use",
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
}
)
content.append(
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
WebSearchTransformation.build_web_search_outcome_block(
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
response: Final[dict[str, object]] = {
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
len(search_result_text),
native_tool is not None,
emit_native_blocks,
)
return response

View file

@ -412,6 +412,7 @@ class WebSearchTransformation:
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: SearchResponse | None,
with_snippets: bool = True,
) -> dict[str, object]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
@ -445,6 +446,8 @@ class WebSearchTransformation:
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
with_snippets: False when a text block in the same reply already
carries the snippets.
"""
items: Final[list[dict[str, object]]] = []
if search_response is not None:
@ -460,7 +463,7 @@ class WebSearchTransformation:
"title": title,
"page_age": page_age,
"encrypted_content": "",
"snippet": getattr(r, "snippet", "") or "",
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
}
)
return {
@ -481,12 +484,15 @@ class WebSearchTransformation:
}
@staticmethod
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
def build_web_search_outcome_block(
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
) -> dict[str, object]:
match outcome:
case SearchSucceeded(response=response):
return WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=response,
with_snippets=with_snippets,
)
case SearchFailed(error_code=error_code):
return WebSearchTransformation.build_web_search_tool_result_error_block(

View file

@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
return getattr(chunk, "usage", None)
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
def _numeric(value: object) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
reading litellm's bill back as if the upstream had reported it."""
hidden: Final[object] = getattr(response, "_hidden_params", None)
if isinstance(hidden, dict):
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
upstream: Final = (
hidden_params[UPSTREAM_USAGE_COST]
if UPSTREAM_USAGE_COST in hidden_params
else _numeric(getattr(usage, "cost", None))
)
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
setattr(usage, "cost", bill)
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
return None
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
cost: Final = (
_numeric(hidden[UPSTREAM_USAGE_COST])
if hidden is not None and UPSTREAM_USAGE_COST in hidden
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
)
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None

View file

@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
Values outside the range fall back to the default and warn, so a misconfigured knob can
neither crash the caller nor silently change the meaning of what it computes.
"""
value: Final = get_env_int(env_var, default)
return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
"""Float counterpart of `get_env_int_in_range`."""
return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
if minimum <= value <= maximum:
return value
logging.getLogger("LiteLLM").warning(

View file

@ -37,6 +37,7 @@ from ..exceptions import (
ServiceUnavailableError,
Timeout,
UnprocessableEntityError,
error_response_without_credentials,
)
@ -305,6 +306,16 @@ def _litellm_proxy_response(
)
def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
if not isinstance(response, httpx.Response):
return response
return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
return response if response is not None and (response.headers or response.content) else None
def _map_openai_exception(
*,
model: str,
@ -315,7 +326,7 @@ def _map_openai_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
raise ContentPolicyViolationError(
message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
@ -378,6 +389,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif (
("invalid_request_error" in error_str and "content_policy_violation" in error_str)
@ -449,6 +461,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
response=_upstream_reply(response),
)
elif "Request too large" in error_str:
raise RateLimitError(
@ -479,6 +492,7 @@ def _map_openai_exception(
model=model,
request=_request,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
@ -512,6 +526,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -563,6 +578,7 @@ def _map_openai_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -572,6 +588,7 @@ def _map_openai_exception(
model=model,
request=getattr(original_exception, "request", None),
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2019,6 +2036,7 @@ def _map_azure_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _without_credentials(getattr(original_exception, "response", None))
message = get_error_message(error_obj=original_exception)
if message is None:
if hasattr(original_exception, "message"):
@ -2054,7 +2072,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "DeploymentNotFound" in error_str:
raise NotFoundError(
@ -2062,7 +2080,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
error_str
@ -2076,6 +2094,7 @@ def _map_azure_exception(
model=model,
extra_information=extra_information,
original_exception=original_exception,
response=response,
)
elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
helpful_message: Final = (
@ -2094,7 +2113,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif getattr(original_exception, "status_code", None) in (None, 400) and (
@ -2106,7 +2125,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
@ -2115,7 +2134,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
@ -2124,7 +2143,7 @@ def _map_azure_exception(
llm_provider=custom_llm_provider,
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "Connection error" in error_str:
raise APIConnectionError(
@ -2140,7 +2159,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif original_exception.status_code == 401:
@ -2149,7 +2168,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 404:
raise NotFoundError(
@ -2157,7 +2176,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 408:
raise Timeout(
@ -2165,6 +2184,7 @@ def _map_azure_exception(
model=model,
litellm_debug_info=extra_information,
llm_provider="azure",
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -2172,7 +2192,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 429:
raise RateLimitError(
@ -2180,7 +2200,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 502:
raise BadGatewayError(
@ -2188,7 +2208,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 503:
raise ServiceUnavailableError(
@ -2196,7 +2216,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 504: # gateway timeout error
raise Timeout(
@ -2205,6 +2225,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
llm_provider="azure",
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -2214,6 +2235,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
model=model,
request=httpx.Request(method="POST", url="https://openai.com/"),
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2589,17 +2611,20 @@ def exception_type(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(
_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
),
)
if (
custom_llm_provider == "litellm_proxy"
): # handle special case where calling litellm proxy + exception str contains error message
extract_and_raise_litellm_exception(
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
error_str=error_str,
model=model,
custom_llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
body=getattr(mappable_exception, "body", None),
)
if (
custom_llm_provider == "openai"

View file

@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

View file

@ -292,6 +292,10 @@ def get_llm_provider(
elif endpoint == "api.deepinfra.com/v1/openai":
custom_llm_provider = "deepinfra"
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
elif endpoint == "api.clfaigateway.dev/v1":
custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch
clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
dynamic_api_key = api_key or clf_env_key
elif endpoint == "api.mistral.ai/v1":
custom_llm_provider = "mistral"
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "clf_ai_gateway":
(
api_base, # rebind-ok: resolved base/key are what this function returns, like every branch
dynamic_api_key, # rebind-ok: see above
) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "empower":
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")

View file

@ -3,6 +3,8 @@ Helper functions for health check calls.
"""
import base64
import itertools
import re
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
)
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
)
@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
return native(model, custom_llm_provider)
def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
try:
model_info: Final = ProviderConfigManager.get_provider_model_info(
model=None, provider=LlmProviders(custom_llm_provider)
)
return None if model_info is None else model_info.get_base_model(model)
except (ValueError, NotImplementedError):
return None
def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
if custom_llm_provider == "bedrock":
from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
return base_model is not None and not base_model.startswith(f"{segment}/")
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
provider_prefix: Final = f"{custom_llm_provider}/"
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
route_segments: Final = tuple(
itertools.takewhile(
lambda segment_index: _is_route_segment(
custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
),
range(len(segments) - 1),
)
)
route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
ranked_model_ids: Final = tuple(
(ranked_model.model.removeprefix(provider_prefix), ranked_model)
for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
)
served_model_ids: Final = sorted(
(ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
)
served_probes: Final = itertools.islice(
dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
)
provider_probes: Final = (
f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
)
return tuple(dict.fromkeys((*served_probes, *provider_probes)))
class HealthCheckHelpers:
@staticmethod
async def ahealth_check_wildcard_models(
@ -99,23 +160,16 @@ class HealthCheckHelpers:
litellm_logging_obj: "Logging",
) -> dict:
from litellm import acompletion
from litellm.litellm_core_utils.llm_request_utils import (
pick_cheapest_chat_models_from_llm_provider,
)
# this is a wildcard model, we need to pick a random model from the provider
cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
if len(cheapest_models) == 0:
probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
if len(probe_models) == 0:
raise Exception(
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
"no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
)
if len(cheapest_models) > 1:
fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list
else:
fallback_models = None
model_params["model"] = cheapest_models[0]
model_params["model"] = probe_models[0]
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["fallbacks"] = list(probe_models[1:]) or None
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
await acompletion(**model_params)
return {}

View file

@ -202,6 +202,7 @@ from litellm.types.utils import (
TextCompletionResponse,
TranscriptionResponse,
Usage,
batch_chat_usage,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
@ -2091,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
if isinstance(priced_result, LiteLLMBatch) and (
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
):
return 0.0
# Fallback: extract router_model_id from litellm_params when not available
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
@ -3346,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
finally:
self._restore_correlation_context()
def _is_batch_poll_without_logging(self, result: object) -> bool:
litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
def _success_handler_body(
self,
result: object = None,
@ -3860,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
## CALCULATE COST FOR BATCH JOBS
if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
litellm_params = self.litellm_params or {}
litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
if (
litellm_metadata.get("batch_ignore_default_logging", False) is True
): # polling job will query these frequently, don't spam db logs
if self._is_batch_poll_without_logging(result):
return
batch_cost: Final = kwargs.get("batch_cost", None)
@ -7481,7 +7487,8 @@ def get_standard_logging_object_payload(
# Extract usage as a plain dict, avoiding Pydantic round-trip
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
)
usage_dict: Final = (
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}

View file

@ -1,6 +1,11 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
@ -151,21 +156,54 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
Returns:
list[str]: A list of the n cheapest chat models.
"""
return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]]
@dataclass(frozen=True, slots=True)
class RankedChatModel:
model: str
route_rank: int
cost: float
is_past_deprecation_date: bool
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
from litellm.utils import bundled_provider_models
if custom_llm_provider not in litellm.models_by_provider:
return []
known_models: Final = bundled_provider_models(custom_llm_provider)
ranked_rows: Final = (
(route_rank, cost, model)
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
)
for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider)
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
)
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost)))
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
class _CostMapDeprecation(TypedDict):
deprecation_date: NotRequired[ReadOnly[object]]
_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
)
except ValueError:
return False
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
for model in models:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
@ -173,7 +211,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
continue
if model_info.get("mode") != "chat":
continue
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
yield (
model,
(model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0),
_is_past_deprecation_date(model_info["key"]),
)
def _route_rank(model: str, custom_llm_provider: str) -> int | None:

View file

@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types

View file

@ -912,9 +912,9 @@ def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Create an Anthropic content block from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
@ -933,14 +933,7 @@ def create_anthropic_image_param(
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
@ -953,14 +946,7 @@ def create_anthropic_image_param(
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
_anthropic_image_param,
)
)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
@ -5878,38 +5869,36 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
Returns the prompt str that's passed to the model as a user message
"""
custom_prompt_details: dict | None = None
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict[
f"{model}/response_schema_prompt"
] # allow user to define custom response schema prompt by model
elif "response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
model_prompt_key: Final = f"{model}/response_schema_prompt"
custom_prompt_details: Final = prompts.get(
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
)
if custom_prompt_details is None:
return default_response_schema_prompt(response_schema=response_schema)
return render_custom_prompt_template(
template=custom_prompt_details,
messages=[{"role": "user", "content": f"{response_schema}"}],
model=model,
llm_provider=llm_provider,
)
def default_response_schema_prompt(response_schema: dict) -> str:
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
"""
Used if provider/model doesn't support 'response_schema' param.
This is the default prompt. Allow user to override this with a custom_prompt.
"""
prompt_str: Final = f"""Use this JSON schema:
```json
{response_schema}
```json
{json.dumps(response_schema, default=str)}
```"""
return prompt_str

View file

@ -1,5 +1,6 @@
import asyncio
import json
import time
import traceback
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.llms.base_llm.realtime.transformation import (
BaseRealtimeConfig,
RealtimeBackend,
RealtimeSessionUpdateRejection,
)
from litellm.types.llms.openai import (
OpenAIRealtimeEvents,
OpenAIRealtimeOutputItemDone,
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_MERGE_MAX_DEPTH: Final = 8
class _SessionUpdateFrame(TypedDict):
type: ReadOnly[str]
session: ReadOnly[Mapping[str, object]]
def _merge_session_fields(
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
) -> Mapping[str, object]:
def merged_value(key: str) -> object:
if key not in update:
return base[key]
value: Final = update[key]
base_value: Final = base.get(key)
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
return _merge_session_fields(
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
)
return update[key]
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
def _session_update_frame(session: Mapping[str, object]) -> str:
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
code="session_update_too_large",
message=(
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
"so this session.update was not applied"
),
param="session",
)
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": "audio_input_not_screenable",
"message": (
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
),
"param": None,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
def _decode_json_object(payload: str) -> Mapping[str, object]:
"""Decode a realtime frame into its top-level field mapping."""
return json.loads(payload)
@ -243,8 +309,17 @@ class RealTimeStreaming:
event_normalizer: RealtimeEventNormalizer | None = None,
usage_meter: RealtimeUsageMeter | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
clock: Callable[[], float] = time.monotonic,
):
self.websocket: _ClientWebSocket = websocket
self._clock: Final = clock
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
self._held_client_content: tuple[str, ...] = ()
self._held_client_content_bytes: int = 0
self._held_client_content_deadline: float | None = None
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
@ -300,6 +375,10 @@ class RealTimeStreaming:
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
# content (realtimeInput / clientContent / toolResponse) has been sent.
self._content_sent_after_setup: bool = False
self._held_session_update: Mapping[str, object] | None = None
self._applied_session_fields: Mapping[str, object] | None = None
self._held_session_update_bytes: int = 0
self._unscreened_audio_refused: bool = False
# Whether this is a transcription-only session (session.type == "transcription",
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
# their input_audio_transcription.completed usage drives duration-based cost.
@ -337,6 +416,17 @@ class RealTimeStreaming:
]
)
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
_CLIENT_CONTENT_TYPES = frozenset(
[
"conversation.item.create",
"response.create",
"input_audio_buffer.append",
"input_audio_buffer.commit",
"input_audio_buffer.end",
]
)
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@ -377,7 +467,7 @@ class RealTimeStreaming:
message_obj: dict[str, Any] = cast(dict[str, Any], message)
else:
message_obj = cast(dict[str, Any], json.loads(message))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
self._collect_tool_calls_from_response_done(message_obj)
if not self._should_store_message(message_obj):
return
try:
@ -502,7 +592,7 @@ class RealTimeStreaming:
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
if event_obj.get("type") != "response.done":
@ -607,6 +697,167 @@ class RealTimeStreaming:
self._usage_meter.observe_client_event(sent_event)
return True
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
return self.provider_config.builds_setup_from_session_update()
def _holds_session_updates_until_first_content(self) -> bool:
return self.session_configuration_request is None and self._builds_setup_lazily()
async def _rejected_setup_session_update(self, message: str) -> bool:
provider_config: Final = self.provider_config
if provider_config is None or not provider_config.builds_setup_from_session_update():
return False
client_event: Final = _decode_json_object(message)
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
if self._leaves_applied_setup_unchanged(update):
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
return True
rejection: Final = provider_config.setup_session_update_rejection(
self.model, update, setup_sent=self.session_configuration_request is not None
)
if rejection is None:
return False
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
return True
def _held_session_update_message(self) -> str:
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
async def _hold_session_update(self, message: str) -> None:
client_event: Final = _decode_json_object(message)
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
await self.websocket.send_text(
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
)
return
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
await self._acknowledge_session_setup(held)
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
if self.provider_config is None:
return None
preview: Final = self.provider_config.transform_realtime_request(
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
applied: Final = self._applied_session_fields
if self.session_configuration_request is None or applied is None:
return False
applied_setup: Final = self._setup_preview(applied)
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
if self.provider_config is None:
return
setup: Final = self._setup_preview(session)
if setup is None:
return
acknowledged: Final = self.provider_config.transform_session_created_event(
self.model, str(self.logging_obj.litellm_trace_id), setup
)
if acknowledged is None:
return
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
event_str: Final = json.dumps(event)
self.store_message(event_str)
await self._send_event_to_client(event, event_str)
async def _send_held_session_update_as_setup(self) -> None:
held: Final = self._held_session_update
if held is None:
return
message: Final = self._held_session_update_message()
self._held_session_update = None
self._held_session_update_bytes = 0
if await self._send_to_backend(message):
self._applied_session_fields = held
await self._send_held_client_content()
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
held_types: Final = (
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
if self._uses_deferred_backend_setup()
else RealTimeStreaming._HELD_CONTENT_TYPES
)
return (
msg_type in held_types
and self._held_session_update is not None
and self._holds_session_updates_until_first_content()
and not self._pending_messages_until_setup
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
)
def _hold_client_content(self, message: str) -> None:
if self._held_client_content_deadline is None:
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
self._held_client_content = (*self._held_client_content, message)
self._held_client_content_bytes += len(message.encode("utf-8"))
async def _send_held_client_content(self) -> None:
held: Final = self._held_client_content
self._held_client_content = ()
self._held_client_content_bytes = 0
self._held_client_content_deadline = None
for message in held:
await self._send_to_backend(message)
def _setup_deadline(self) -> float:
content_deadline: Final = self._held_client_content_deadline
return (
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
)
async def _send_setup_before_content(self, msg_type: str | None) -> None:
if self._holds_session_updates_until_first_content():
await self._send_held_session_update_as_setup()
await self._send_default_setup_before_first_client_message(msg_type)
async def _receive_client_message_before_setup_deadline(self) -> str | None:
if not self._holds_session_updates_until_first_content():
return await self._receive_client_message()
receive: Final = asyncio.ensure_future(self._receive_client_message())
try:
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
if not done:
await self._send_setup_before_content(None)
return await receive
finally:
receive.cancel()
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
if self.session_configuration_request is not None or self.provider_config is None:
return
if msg_type == "session.update" or (
self._uses_deferred_backend_setup()
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
):
return
default_setup: Final = self.provider_config.session_configuration_request(self.model)
if not default_setup:
return
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
await self.backend_ws.send(setup)
self._cache_session_configuration_request(setup)
self._applied_session_fields = _NO_SESSION_FIELDS
await self._send_held_client_content()
def _enforce_transcription_session_model(self, message: str) -> str:
"""Force client transcription session updates to the authorized model.
@ -851,6 +1102,20 @@ class RealTimeStreaming:
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
if pre_block_backend_message is None and self._is_transcription_session:
return
await self._send_setup_before_content("conversation.item.create")
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
@ -958,6 +1223,21 @@ class RealTimeStreaming:
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
return (
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
and self._builds_setup_lazily()
and self._should_disable_vad_auto_response()
)
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
return
self._unscreened_audio_refused = True
await self.websocket.send_text(
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
)
async def run_realtime_guardrails(
self,
transcript: str,
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
break
finally:
self._flushing_pending_messages_until_setup = False
if self.session_created_sent_to_client and self._builds_setup_lazily():
await self._maybe_send_guardrail_turn_detection_update()
continue
if self.session_created_sent_to_client:
# A synthetic session.created (with placeholder defaults) was
# already forwarded to the client when we connected. The
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
return (
close.code != 1000
and not self._backend_sent_frames
and not self._holds_session_updates_until_first_content()
)
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
client_event: _ClientEventFrame
try:
while True:
message = await self._receive_client_message()
message = await self._receive_client_message_before_setup_deadline()
if message is None:
return ClientLoopExit.CLIENT_DISCONNECTED
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
if (
msg_type == "session.update"
and self.session_configuration_request is None
and self._held_session_update is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
and (
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
except (json.JSONDecodeError, AttributeError):
pass
if self._refuses_unscreened_audio(msg_type):
await self._refuse_unscreened_audio(msg_type, message)
continue
## LOGGING
# Log after any in-place modifications (GA remap, guardrail
# turn_detection injection) so audit logs reflect what we
# actually forward to the backend.
self.store_input(message=message)
if self._holds_client_content(msg_type, message):
self._hold_client_content(message)
continue
if self._held_client_content and msg_type != "session.update":
await self._send_setup_before_content(msg_type)
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
await self._send_setup_before_content(msg_type)
if self._should_buffer_client_message_until_setup(message):
self._buffer_pending_message_until_setup(message)
continue
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
continue
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
await self._hold_session_update(message)
if self._pending_messages_until_setup:
await self._send_held_session_update_as_setup()
continue
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
await self._send_setup_before_content(msg_type)
try:
sent = await self._send_to_backend(message)
except RealtimeTranscriptionProtocolError as e:

View file

@ -3,7 +3,7 @@ import builtins
import time
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
from itertools import chain, groupby
from itertools import chain, groupby, takewhile
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import (
USAGE_COST_BILLED_PROVIDERS,
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
chunk_usage,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
@ -257,7 +258,7 @@ def apply_grounding_request_counts(
class ChunkProcessor:
def __init__(self, chunks: list, messages: Sequence | None = None):
def __init__(self, chunks: list, messages: Sequence[object] | None = None):
self.chunks = self._sort_chunks(chunks)
self.messages = messages
self.first_chunk = chunks[0]
@ -390,6 +391,30 @@ class ChunkProcessor:
case _:
return "assistant"
@staticmethod
def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object:
finish_reason = unreported
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
return finish_reason
@staticmethod
def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices:
return Choices(
index=index,
finish_reason=ChunkProcessor._finish_reason_of_chunks(
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
),
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
)
@staticmethod
def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
"""
@ -414,7 +439,11 @@ class ChunkProcessor:
None,
)
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
def build_base_response(
self,
chunks: Sequence["_BaseChunk"],
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
) -> ModelResponse:
chunk = self.first_chunk
id: Final = ChunkProcessor._get_chunk_id(chunks)
object: Final = chunk["object"]
@ -426,16 +455,7 @@ class ChunkProcessor:
service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks)
role: Final = ChunkProcessor._get_role_from_chunks(chunks)
finish_reason = "stop"
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
# Initialize the response dictionary
response = ModelResponse(
@ -455,13 +475,17 @@ class ChunkProcessor:
if service_tier is not None
else MappingProxyType({})
),
"choices": [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
],
"choices": (
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
if len(choice_streams) > 1
else [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
]
),
"usage": {
"prompt_tokens": 0, # Modify as needed
"completion_tokens": 0, # Modify as needed
@ -1357,9 +1381,6 @@ def stream_chunk_builder_text_completion(
return TextCompletionResponse(**response)
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return None
@ -1367,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
return provider
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
@ -1381,9 +1398,6 @@ def _stream_builder_response_cost(
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
@ -1430,13 +1444,15 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
return
setattr(usage, "cost", computed_cost)
_set_hidden_response_cost(response, float(computed_cost))
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
def _field_of(value: object, name: str) -> object:
@ -1520,11 +1536,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
def _stream_choice_index(choice: object) -> int:
index: Final = _field_of(choice, "index")
return index if isinstance(index, int) else 0
def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool:
return any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"]
for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks)
)
def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]:
indices: Final = sorted(
frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks)))
)
if len(indices) <= 1:
return ((0, chunks),)
return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices)
def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]:
return tuple(
_BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index])
for chunk in chunks
)
def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None:
choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices"))
parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk)))
return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None
def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]:
tool_call_chunks: Final = [
chunk
for chunk in chunks
if any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
for choice in chunk.get("choices") or ()
)
]
if tool_call_chunks:
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
yield "content", None
yield "tool_calls", tool_calls_list
function_call_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "function_call" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["function_call"] is not None
]
if function_call_chunks:
yield "content", None
yield "function_call", processor.get_combined_function_call_content(function_call_chunks)
content_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["content"] is not None
]
if content_chunks:
yield "content", processor.get_combined_content(content_chunks)
thinking_blocks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "thinking_blocks" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
]
if thinking_blocks:
yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks)
reasoning_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "reasoning_content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
]
if reasoning_chunks:
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
yield "reasoning_items", list(reasoning_items)
annotation_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "annotations" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["annotations"] is not None
]
message_annotations, provider_annotations = (
partition_annotations(
list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks))
)
if annotation_chunks
else (None, ())
)
if message_annotations is not None:
yield "annotations", message_annotations
audio_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "audio" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["audio"] is not None
]
if audio_chunks:
yield "audio", processor.get_combined_audio_content(audio_chunks)
# Handle image chunks from models like gemini-2.5-flash-image
# See: https://github.com/BerriAI/litellm/issues/19478
image_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "images" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["images"] is not None
]
if image_chunks:
# Images come complete in a single chunk, collect all images from all chunks
all_images: Final = []
for chunk in image_chunks:
all_images.extend(chunk["choices"][0]["delta"]["images"])
yield "images", all_images
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
# See: https://github.com/BerriAI/litellm/issues/17737
provider_specific_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "provider_specific_fields" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
]
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {}
combined_provider_fields: Final = {
key: value
for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
yield "provider_specific_fields", combined_provider_fields
def stream_chunk_builder(
chunks: list,
messages: Sequence | None = None,
start_time=None,
end_time=None,
start_time: object = None,
end_time: object = None,
logging_obj: Optional["Logging"] = None,
count_prompt_tokens: Callable[[], int] | None = None,
) -> ModelResponse | TextCompletionResponse | None:
@ -1561,26 +1754,15 @@ def stream_chunk_builder(
usage_model: Final = usage_tokenizer_model(
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
)
# Initialize the response dictionary
response: Final = processor.build_base_response(chunks)
choice_streams: Final = _choice_streams(chunks)
response: Final = processor.build_base_response(chunks, choice_streams)
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
# Fast path for the common text-only streaming case:
# avoid repeated multi-pass list scans over chunks.
simple_content_parts: Final[list[str]] = []
is_simple_text_stream = True
for chunk in chunks:
if not chunk.get("choices"):
continue
if (part := _simple_text_part(chunk["choices"])) is None:
is_simple_text_stream = False
break
if part:
simple_content_parts.append(part)
if is_simple_text_stream:
if simple_content_parts:
response["choices"][0]["message"]["content"] = "".join(simple_content_parts)
simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams)
if None not in simple_texts:
for (choice, _), simple_text in zip(rebuilt_choices, simple_texts):
if simple_text:
choice.message.content = simple_text
completion_output = get_content_from_model_response(response)
usage = processor.calculate_usage(
chunks=chunks,
@ -1619,165 +1801,9 @@ def stream_chunk_builder(
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
tool_call_chunks: Final = [
chunk
for chunk in chunks
if any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
for choice in chunk.get("choices") or ()
)
]
if len(tool_call_chunks) > 0:
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
_choice = response.choices[0]
_choice.message.content = None
_choice.message.tool_calls = tool_calls_list
function_call_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "function_call" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["function_call"] is not None
]
if len(function_call_chunks) > 0:
_choice = response.choices[0]
_choice.message.content = None
_choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks)
content_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["content"] is not None
]
if len(content_chunks) > 0:
response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks)
thinking_blocks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "thinking_blocks" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
]
if len(thinking_blocks) > 0:
response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content(
thinking_blocks
)
reasoning_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "reasoning_content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
]
if len(reasoning_chunks) > 0:
response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content(
reasoning_chunks
)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
annotation_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "annotations" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["annotations"] is not None
]
if len(annotation_chunks) > 0:
# Merge annotations from ALL chunks — providers may spread
# them across multiple streaming chunks or send them only in
# the final chunk.
all_annotations: Final[list[object]] = []
for ac in annotation_chunks:
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
message_annotations, provider_annotations = partition_annotations(all_annotations)
if message_annotations is not None:
response["choices"][0]["message"]["annotations"] = message_annotations
if provider_annotations:
response.choices[0].message.provider_specific_fields = {
**(response.choices[0].message.provider_specific_fields or {}),
"annotations": list(provider_annotations),
}
audio_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "audio" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["audio"] is not None
]
if len(audio_chunks) > 0:
_choice = response.choices[0]
_choice.message.audio = processor.get_combined_audio_content(audio_chunks)
# Handle image chunks from models like gemini-2.5-flash-image
# See: https://github.com/BerriAI/litellm/issues/19478
image_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "images" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["images"] is not None
]
if len(image_chunks) > 0:
# Images come complete in a single chunk, collect all images from all chunks
all_images: Final = []
for chunk in image_chunks:
all_images.extend(chunk["choices"][0]["delta"]["images"])
response["choices"][0]["message"]["images"] = all_images
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
# See: https://github.com/BerriAI/litellm/issues/17737
provider_specific_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "provider_specific_fields" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
]
if len(provider_specific_chunks) > 0:
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = (
{"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
)
combined_provider_fields: Final = {
key: value
for fields in (citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
_choice = response.choices[0]
_choice.message.provider_specific_fields = {
**(_choice.message.provider_specific_fields or {}),
**combined_provider_fields,
}
for choice, choice_chunks in rebuilt_choices:
for field, value in _assembled_message_fields(processor, choice_chunks):
choice.message[field] = value
completion_output = get_content_from_model_response(response)

View file

@ -7,14 +7,17 @@ import re
import struct
from bisect import bisect_right
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from dataclasses import dataclass
from functools import cached_property
from itertools import accumulate, chain, islice
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -33,7 +36,12 @@ from litellm.constants import (
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
MAX_TILE_HEIGHT,
MAX_TILE_WIDTH,
OPENAI_CHAT_MODES,
OPENAI_MODEL_NAME_MAX_DASHES,
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
PDF_DATA_URL_PREFIX,
PDF_MAX_FORM_XOBJECT_DEPTH,
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
TOKEN_COUNTER_MAX_EXACT_CHARS,
@ -62,6 +70,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
AnthropicSearchResultParam,
CompactionBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -606,6 +615,35 @@ TokenCounterFunction = Callable[[str], int]
Type for a function that counts tokens in a string.
"""
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
class _PdfBox(Protocol):
@property
def width(self) -> float: ...
@property
def height(self) -> float: ...
class _PdfContent(Protocol):
def get_data(self) -> bytes: ...
class _PdfPage(Protocol):
@property
def mediabox(self) -> _PdfBox: ...
def extract_text(self) -> str: ...
def get_contents(self) -> _PdfContent | None: ...
class _PdfReader(Protocol):
@property
def pages(self) -> Sequence[_PdfPage]: ...
EXTRAPOLATION_SAMPLES: Final = 16
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
@ -662,7 +700,7 @@ class _MessageCountParams:
if uses_legacy_message_accounting(model):
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
self.tokens_per_name = -1 # if there's a name, the role is omitted
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
elif _is_openai_chat_model(actual_model):
self.tokens_per_message = 3
self.tokens_per_name = 1
else:
@ -671,6 +709,63 @@ class _MessageCountParams:
self.tokens_per_name = 1
self.count_function = _get_count_function(model, custom_tokenizer)
self.tool_section = chat_template_tool_section(model)
self._model = model
@cached_property
def pdf_pricing(self) -> PdfPagePricing:
return pdf_page_pricing(self._model)
def _is_openai_chat_model(model: str) -> bool:
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
def _openai_deployment_name(model: str) -> str:
name: Final = model.rpartition("/")[2]
return name.split(":")[1] if name.startswith("ft:") else name
def _openai_model_family(model: str) -> str | None:
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
return None if listed is None or _is_listed_non_chat_model(listed) else listed
def _untyped_cost_map() -> object:
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
def _cost_map_entry(model: str) -> Mapping[str, object]:
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
return _pdf_dictionary(cost_map.get(model)) or {}
def _is_azure_openai_chat_model(model: str) -> bool:
name: Final = _openai_deployment_name(model)
entry: Final = _cost_map_entry(f"azure/{name}")
return (
entry.get("litellm_provider") == "azure"
and entry.get("mode") in OPENAI_CHAT_MODES
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
)
def _is_openai_or_azure_model(model: str) -> bool:
if _is_listed_non_chat_model(_openai_deployment_name(model)):
return False
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
def reads_pdf_pages(model: str) -> bool:
segments: Final = model.split("/")
names: Final = ("/".join(segments[start:]) for start in range(len(segments)))
stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names)
return next((support for support in stated if support is not None), False) is True
def token_counter(
@ -842,7 +937,10 @@ def _count_messages(
value,
use_default_image_token_count,
default_token_count,
params.pdf_pricing,
)
elif key == "provider_specific_fields":
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
@ -857,6 +955,25 @@ def _count_messages(
return num_tokens
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
fields: Final = as_str_mapping(provider_specific_fields)
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
block_fields: Final = (
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
)
return tuple(
content
for block in block_fields
if block is not None
and block.get("type") == "compaction"
and isinstance(content := block.get("content"), str)
and content
)
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
@ -986,7 +1103,7 @@ def _get_exact_count_function(
return count_tokens
if tokenizer_json["type"] == "openai_tokenizer":
encoding: Final = openai_tokenizer_encoding(model)
encoding: Final = openai_tokenizer_encoding(model or "")
def encode_length(text: str) -> int:
return _encoding_count(encoding, text)
@ -1006,6 +1123,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
return get_encoding(openai_tokenizer_encoding_name(model))
def _is_listed_non_chat_model(model: str) -> bool:
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding.
@ -1021,7 +1143,9 @@ def openai_tokenizer_encoding_name(model: str) -> str:
try:
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
return _suffixed_variant_encoding_name(model_to_use) or (
"cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
)
def unresolved_tokenizer_model(model: str) -> str:
@ -1079,7 +1203,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
def _openai_model_name(model: str) -> str | None:
return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
return (
_listed_openai_model_name(model)
or _tiktoken_o200k_model_name(model)
or _versioned_openai_model_name(model)
or _azure_openai_chat_model_name(model)
)
def _azure_openai_chat_model_name(model: str) -> str | None:
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None
_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
@ -1249,7 +1382,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
def _pdf_value(container: Mapping[str, object], key: str) -> object:
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
_PdfXObjects: TypeAlias = Mapping[str, object] | None
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfContentSummary:
drawn_names: tuple[str, ...]
draws_inline_image: bool
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfWalk:
frontier: tuple[_PdfDrawing, ...]
seen: frozenset[tuple[int, int]]
summaries: Mapping[int, _PdfContentSummary | None]
draws: int
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
if resources is None:
return inherited
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
_PDF_XOBJECT_DRAW: Final = re.compile(
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
)
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
def _decoded(raw: bytes, charset: str) -> str | None:
try:
return raw.decode(charset)
except UnicodeDecodeError:
return None
def _pdf_name(escaped: bytes) -> str:
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
data: Final = content.get_data()
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
return _PdfContentSummary(
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
)
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
from pypdf.generic import StreamObject
if not isinstance(form, StreamObject):
return None
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
summary, xobjects = drawing
if xobjects is None:
return ()
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
xobject, inherited = use
return id(xobject), id(_pdf_xobjects(xobject, inherited))
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
return None
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
return None
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
summaries: Final = MappingProxyType(
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
)
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
return _PdfWalk(
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
seen=walk.seen | frozenset(forms),
summaries=summaries,
draws=draws,
)
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
contents: Final = page.get_contents()
page_dictionary: Final = _pdf_dictionary(page)
if contents is None or page_dictionary is None:
return False
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
seen=frozenset(),
summaries=MappingProxyType({}),
draws=0,
)
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
if not walk.frontier:
return False
if any(summary.draws_inline_image for summary, _ in walk.frontier):
return True
if (next_walk := _pdf_next_walk(walk)) is None:
return True
walk = next_walk
return bool(walk.frontier)
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
if _pdf_page_draws_raster_image(page):
return _claude_pdf_page_tokens(page, count_function)
text: Final = page.extract_text() or ""
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
float(page.mediabox.width), float(page.mediabox.height)
)
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
match pdf_pricing:
case "openai":
return _openai_pdf_page_tokens(page, count_function)
case "claude":
return _claude_pdf_page_tokens(page, count_function)
def _count_inline_pdf_tokens(
data_url: str,
count_function: TokenCounterFunction,
pdf_pricing: PdfPagePricing = "claude",
) -> int | None:
if not data_url.startswith(PDF_DATA_URL_PREFIX):
return None
try:
@ -1258,12 +1549,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
return None
try:
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
return sum(
count_function(page.extract_text() or "")
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
for page in reader.pages
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
)
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
except Exception as e:
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
return None
@ -1273,8 +1562,9 @@ def _count_opaque_document_tokens(
data_url: str,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
if pdf_tokens is not None:
return pdf_tokens
return calculate_img_tokens(
@ -1289,6 +1579,7 @@ def _count_document_tokens(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
source: Final = document["source"]
metadata_tokens: Final = sum(
@ -1301,10 +1592,10 @@ def _count_document_tokens(
if isinstance(content, str):
return metadata_tokens + count_function(content)
return metadata_tokens + _count_content_list(
count_function, content, use_default_image_token_count, default_token_count
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
)
return metadata_tokens + _count_opaque_document_tokens(
_anthropic_image_source_data(source), count_function, use_default_image_token_count
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
)
@ -1312,6 +1603,7 @@ def _count_file_tokens(
file_value: object,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
if not isinstance(file_value, Mapping):
@ -1321,7 +1613,9 @@ def _count_file_tokens(
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
if not isinstance(file_data, str) or not file_data:
return name_tokens
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
return name_tokens + _count_opaque_document_tokens(
file_data, count_function, use_default_image_token_count, pdf_pricing
)
def _count_anthropic_content(
@ -1329,6 +1623,7 @@ def _count_anthropic_content(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
@ -1363,6 +1658,7 @@ def _count_anthropic_content(
field_value,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif isinstance(field_value, dict):
tokens += count_function(str(field_value))
@ -1375,6 +1671,11 @@ def _count_anthropic_content(
return tokens
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1386,9 +1687,11 @@ def _count_content_list(
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| AnthropicSearchResultParam
| CompactionBlock
],
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""Recursively count tokens from a list of content blocks."""
try:
@ -1396,8 +1699,10 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
if block_text:
num_tokens += count_function(block_text)
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1413,12 +1718,14 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] == "file":
num_tokens += _count_file_tokens(
c.get("file"),
count_function,
use_default_image_token_count,
pdf_pricing,
)
elif c["type"] in ("tool_use", "tool_result", "search_result"):
num_tokens += _count_anthropic_content(
@ -1426,13 +1733,8 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -1450,7 +1752,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"compaction, tool_reference, search_result)."
)
return num_tokens
except Exception as e:

View file

@ -14,6 +14,7 @@ import json
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Optional
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -28,6 +29,8 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
class _A2ATextPart(TypedDict, total=False):
"""The subset of an A2A message part this handler reads text from."""
@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation):
obj = item
elif isinstance(item, str):
try:
obj = json.loads(item.strip())
except (json.JSONDecodeError, TypeError):
obj = _JSON_OBJECT.validate_python(json.loads(item.strip()))
except (json.JSONDecodeError, TypeError, ValidationError):
continue
else:
continue

View file

@ -18,13 +18,13 @@ from copy import deepcopy
from dataclasses import dataclass
from itertools import chain, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
unappliable_request_rewrite,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
AllAnthropicToolsValues,
AnthropicMessagesRequest,
)
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
text: str
@runtime_checkable
class _WritableMessage(Protocol):
@overload
def get(self, key: str, /) -> object | None: ...
@ -195,6 +197,140 @@ class _ToolCallShape:
arguments: str
@dataclass(frozen=True, slots=True)
class _WebSearchString:
block_idx: int
result_idx: int | None
field: str
value: str
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
choice: Final = response.choices[0] if response.choices else None
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
blocks: Final = (
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
if fields is not None and isinstance(fields.get("web_search_results"), list)
else ()
)
return tuple(
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
)
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
fields: Final = as_str_mapping(result)
if fields is None or fields.get("type") != "web_search_result":
return ()
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
return tuple(
_WebSearchString(block_idx, result_idx, field, value)
for field, value in values
if isinstance(value, str) and value
)
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
block_id: Final = block.get("id")
match block.get("type"):
case "server_tool_use":
return True
case "tool_use":
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
case _:
return False
def _web_search_block_strings(
block_idx: int, block: Mapping[str, object], *, with_query: bool
) -> tuple[_WebSearchString, ...]:
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
``web_search_tool_result`` result's title and url."""
match block.get("type"):
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
tool_input: Final = as_str_mapping(block.get("input"))
query: Final = None if tool_input is None else tool_input.get("query")
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
case "web_search_tool_result":
return tuple(
chain.from_iterable(
_web_search_result_strings(block_idx, result_idx, result)
for result_idx, result in enumerate(_web_search_results(block))
)
)
case _:
return ()
def _rewritten_web_search_result(
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
) -> object:
fields: Final = as_str_mapping(result)
if fields is None:
return result
return {
**fields,
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
}
def _rewritten_web_search_block(
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
) -> Mapping[str, object]:
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
if _is_server_tool_use(block):
query: Final = by_location.get((None, "query"))
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
return block if query is None else {**block, "input": {**tool_input, "query": query}}
if not isinstance(block.get("content"), list):
return block
return {
**block,
"content": [
_rewritten_web_search_result(result_idx, result, by_location)
for result_idx, result in enumerate(_web_search_results(block))
],
}
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
block: Final = as_str_mapping(content_block)
if block is None or not isinstance(content_block, _WritableMessage):
return
rewritten: Final = _rewritten_web_search_block(block, rewrites)
writable: Final = _as_writable(content_block)
for key in ("input", "content"):
if key in rewritten:
writable[key] = rewritten[key]
def _split_web_search_texts(
guardrailed: Sequence[str], web_search_count: int, reply_count: int
) -> tuple[Sequence[str], Sequence[str]]:
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
return guardrailed[:split_at], guardrailed[split_at:]
def _web_search_rewrites_by_block(
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
rewrites: Final = tuple(
(target, value)
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
)
return MappingProxyType(
{
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
for block_idx in {target.block_idx for target, _ in rewrites}
}
)
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
return (
block.get("type") == "tool_use"
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
_DECODED_SSE_DATA: Final = TypeAdapter(object)
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
_RELEASED_TOOL_USE_STOP: Final = (
b"event: message_delta\n"
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
@ -315,6 +452,51 @@ def _write_back_tool_use(
block["name"] = shape.name
def _streamed_query_strings(
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
) -> tuple[_WebSearchString, ...]:
if len(tool_call_blocks) != len(tool_calls):
return ()
return tuple(
chain.from_iterable(
_web_search_block_strings(
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
)
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
)
)
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
return MappingProxyType(
{
target.block_idx: value
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
}
)
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
if arguments is None:
return tool_call
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
def _with_query_rewrites(
tool_calls: tuple[_ToolCallShape, ...],
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
query_rewrites: Mapping[int, str],
) -> tuple[_ToolCallShape, ...]:
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
return tool_calls
return tuple(
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
)
@dataclass(frozen=True, slots=True)
class _SSEFieldRewrite:
"""One field of one nested section of a buffered SSE event, rewritten."""
@ -349,14 +531,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...
)
class _AnthropicSSEDelta(TypedDict, total=False):
type: ReadOnly[str]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSETextDelta(TypedDict, total=False):
type: ReadOnly[object]
text: ReadOnly[str]
stop_reason: ReadOnly[str | None]
class _AnthropicSSEEvent(TypedDict, total=False):
delta: ReadOnly[_AnthropicSSEDelta]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSETextDeltaEvent(TypedDict, total=False):
delta: ReadOnly[_SSETextDelta]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSEStopDelta(TypedDict, total=False):
stop_reason: ReadOnly[object]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSEStopDeltaEvent(TypedDict, total=False):
delta: ReadOnly[_SSEStopDelta]
_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent)
_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent)
_SSEEventT = TypeVar("_SSEEventT")
def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None:
try:
return adapter.validate_python(json.loads(data_line))
except json.JSONDecodeError:
verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type)
except ValidationError:
verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type)
return None
class AnthropicMessagesHandler(BaseTranslation):
@ -376,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
def post_call_hook_response(self, response: object) -> object:
if not isinstance(response, ModelResponse):
return response
return self.adapter.translate_openai_response_to_anthropic(response)
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
result_blocks: Final = _streamed_web_search_result_blocks(response)
if not result_blocks:
return translated
content: Final = translated.get("content") or ()
return {**translated, "content": [*result_blocks, *content]}
@staticmethod
def _build_streaming_usage_response(
@ -1247,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
task_mappings,
tool_calls_to_check,
)
block_fields: Final = tuple(map(self._output_block_fields, response_content))
web_search_strings: Final = tuple(
chain.from_iterable(
_web_search_block_strings(content_idx, fields[1], with_query=True)
for content_idx, fields in enumerate(block_fields)
if fields is not None
)
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_to_check or web_search_strings:
request_data = self._prepare_request_data(
request_data,
response,
@ -1258,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
)
inputs: Final = self._build_guardrail_inputs(
texts_to_check,
[*(target.value for target in web_search_strings), *texts_to_check],
images_to_check,
tool_calls_to_check,
response,
@ -1271,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
web_search_rewrites, reply_rewrites = _split_web_search_texts(
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
)
# Step 3: Map guardrail responses back to original response structure
await self._apply_guardrail_responses_to_output(
response=response,
responses=guardrailed_texts,
responses=list(reply_rewrites),
task_mappings=task_mappings,
)
self._write_back_web_search_strings(
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
)
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
@ -1324,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
guardrail_inputs: Final = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
if texts_to_check:
guardrail_inputs["texts"] = texts_to_check
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
@ -1348,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
responses_so_far, request_data
)
raise
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
web_search_rewrites, reply_rewrites = _split_web_search_texts(
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
)
if (
deliver_ended_stream_rewrites
and isinstance(string_so_far, str)
and string_so_far
and guardrailed_texts
and guardrailed_texts[0] != string_so_far
and reply_rewrites
and reply_rewrites[0] != string_so_far
):
self._write_ended_stream_text_rewrite(
responses_so_far,
guardrailed_texts[0],
reply_rewrites[0],
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
if deliver_ended_stream_rewrites:
self._write_ended_stream_web_search_rewrites(
responses_so_far,
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
)
if deliver_ended_stream_rewrites:
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
self._write_ended_stream_tool_call_rewrites(
responses_so_far,
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
post_guardrail_tool_calls=_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
post_guardrail_tool_calls=_with_query_rewrites(
_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
),
tool_call_blocks,
_query_rewrites(
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
),
),
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@ -1425,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
request_data["litellm_metadata"] = user_metadata
return request_data
@staticmethod
def _write_back_web_search_strings(
response_content: Sequence[object],
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
for block_idx, rewrites in rewrites_by_block.items():
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
@staticmethod
def _get_response_content(response: object) -> Sequence[object]:
"""Extract content list from a dict or object response."""
@ -1547,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
) -> None:
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
buffered chunks in place: the rebuilt response lists tool calls in the
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
on the nth block, its first ``input_json_delta`` carrying the full rewritten
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
arguments, every later one blanked, and ``content_block_start`` carrying the
rewritten name. Blocks that do not line up with the rebuilt tool calls make
the rewrite undeliverable, so the pipeline executor discards it and releases
the original chunks."""
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
block_indices: Final = tuple(
index
for item in responses_so_far
for event in cls._iter_sse_events(item)
if event.get("type") == "content_block_start"
and isinstance(block := event.get("content_block"), Mapping)
and block.get("type") == "tool_use"
and isinstance(index := event.get("index"), int)
)
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
if len(block_indices) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
f"{len(block_indices)} tool_use blocks",
f"{len(block_indices)} tool_use or server_tool_use blocks",
)
rewrites_by_block: Final = MappingProxyType(
{
@ -1610,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
),
)
@classmethod
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
result_blocks: Final = (
(index, block)
for event in events
if event.get("type") == "content_block_start"
and isinstance(index := event.get("index"), int)
and (block := as_str_mapping(event.get("content_block"))) is not None
)
return tuple(
chain.from_iterable(
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
)
)
@classmethod
def _write_ended_stream_web_search_rewrites(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
if not rewrites_by_block:
return
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
index: Final = event.get("index")
block: Final = as_str_mapping(event.get("content_block"))
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
if event.get("type") != "content_block_start" or block is None or rewrites is None:
return None
return _SSEFieldRewrite(
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
@classmethod
def _streamed_tool_call_blocks(
cls, responses_so_far: Sequence[object]
) -> tuple[tuple[int, Mapping[str, object]], ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
return tuple(
(index, block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and isinstance(block_type := block.get("type"), str)
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
and isinstance(index := event.get("index"), int)
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
@ -1712,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
return StreamingScanKey(
texts=(self.get_streaming_string_so_far(responses_so_far),),
texts=(*(target.value for target in result_strings), streamed_text),
tool_calls=tool_use_fingerprints if stream_ended else (),
stream_ended=stream_ended,
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,
@ -1806,13 +2107,10 @@ class AnthropicMessagesHandler(BaseTranslation):
# Only process content_block_delta events
if event_type == "content_block_delta" and data_line:
try:
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
if delta.get("type") == "text_delta":
text += delta.get("text", "")
except json.JSONDecodeError:
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line)
delta: _SSETextDelta = {} if data is None else data.get("delta", {})
if delta.get("type") == "text_delta":
text += delta.get("text", "")
except Exception as e:
verbose_proxy_logger.error("Error extracting text from SSE: %s", e)
@ -1869,14 +2167,11 @@ class AnthropicMessagesHandler(BaseTranslation):
# Check for message_delta event with stop_reason
if event_type == "message_delta" and data_line:
try:
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
stop_reason: str | None = delta.get("stop_reason")
if stop_reason is not None:
return True
except json.JSONDecodeError:
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line)
delta: _SSEStopDelta = {} if data is None else data.get("delta", {})
stop_reason: object = delta.get("stop_reason")
if stop_reason is not None:
return True
except Exception as e:
verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e)

View file

@ -8,12 +8,13 @@ import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -24,6 +25,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
def _message_blocks(message: object) -> Sequence[object]:
fields: Final = as_str_mapping(message)
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
return blocks or ()
def _is_signed_compaction_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "compaction":
return False
signature: Final = fields.get("signature")
return isinstance(signature, str) and bool(signature)
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
if not isinstance(context_management, list):
return context_management
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
[entry for entry in context_management if isinstance(entry, dict)]
)
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
any(edit_type != "compact_20260112" for edit_type in edit_types),
),
)
if used
)
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drop_params,
)
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

View file

@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)

View file

@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(
self,
auth_header: Mapping[str, str],
messages: Sequence[Mapping[str, JsonValue]],
context_management: JsonValue = None,
) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
merges with the token-counting beta instead of replacing it."""
merges with the token-counting beta instead of replacing it. A history that carries a
signed compaction block also needs the compaction beta, or Anthropic rejects the block
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
compaction_betas: Final = (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
if carries_signed_compaction_block(messages)
else ()
)
request_betas: Final = (
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
*compaction_betas,
*context_management_betas(context_management),
)
return {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
),
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
}
def validate_request(

View file

@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
api_base: Final = extra_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
)

View file

@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
@ -255,7 +256,12 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
self,
kwargs,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -289,6 +295,7 @@ class AnthropicAdapter:
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
)
@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter:
)
@staticmethod
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
def _supports_prompt_cache_key(
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
) -> bool:
if not model or not custom_llm_provider:
return False
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter:
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "prompt_cache_key" in (supported_params or ())
if "prompt_cache_key" not in (supported_params or ()):
return False
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
return True
_, _, _, resolved_api_base = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
mistral_config: Final = litellm.MistralConfig()
return mistral_config.takes_prompt_cache_key_value(
prompt_cache_key
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
@staticmethod
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> None:
"""Translate metadata fields from Anthropic request to OpenAI request."""
if "metadata" in anthropic_message_request:
@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["user"] = metadata["user_id"]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None and self._supports_prompt_cache_key(
anthropic_message_request.get("model"), custom_llm_provider
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
):
new_kwargs["prompt_cache_key"] = prompt_cache_key
@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
## CONVERT TOOL CHOICE
self._translate_tool_choice_to_openai(

View file

@ -12,11 +12,20 @@ import json
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
citations: Final = block.get("citations")
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
class FakeAnthropicMessagesStreamIterator:
"""
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
chunks: Final = list[bytes]()
block_type: Final = block_dict.get("type")
if block_type == "text":
citations: Final = _citations_of(block_dict)
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
"content_block": {"type": "text", "text": "", "citations": ()}
if citations
else {"type": "text", "text": ""},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
citation_deltas: Final = (
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "citations_delta", "citation": citation},
}
for citation in citations
)
chunks.extend(
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
)
text: Final = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
elif block_type == "redacted_thinking":
content_block_start = {
elif block_type in ("tool_use", "server_tool_use"):
empty_input: Final[Mapping[str, object]] = {}
tool_use_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
"content_block": {**block_dict, "input": empty_input},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
input_data: Final = block_dict.get("input", {})
content_block_delta = {
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
tool_use_delta: Final = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
"partial_json": json.dumps(block_dict.get("input", {})),
},
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
else:
passthrough_start: Final = {

View file

@ -43,6 +43,7 @@ from ...common_utils import (
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
context_management_betas,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
if transformed_context_management is not None:
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
elif drops_unsupported_params(litellm_params.drop_params):
anthropic_messages_optional_request_params.pop("context_management")
####### get required params for all anthropic messages requests ######
# Lazy %s: the f-string previously stringified the entire messages
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

View file

@ -42,6 +42,7 @@ from ..openai.common_utils import (
build_output_token_limit_response,
is_output_token_limit_error,
model_dump_provider_response,
upstream_error_response,
)
from .common_utils import (
AzureOpenAIError,
@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
error_text = error_response.text
raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def make_async_azure_httpx_request(
self,
@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
error_code: Final[object] = getattr(e, "status_code", None)
if error_code is not None:
raise AzureOpenAIError(status_code=error_code, message=str(e))
raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e))
else:
raise AzureOpenAIError(status_code=500, message=str(e))

View file

@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
from ...openai.common_utils import upstream_error_response
from ...openai.completion.transformation import OpenAITextCompletionConfig
from ..common_utils import AzureOpenAIError, BaseAzureLLM
@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
async def acompletion(
self,
@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
def streaming(
self,
@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)

View file

@ -1,5 +1,7 @@
from typing import Final
import httpx
from litellm.exceptions import ContentPolicyViolationError
@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping:
model: str,
extra_information: str,
original_exception: Exception,
response: httpx.Response | None,
) -> ContentPolicyViolationError:
"""
Create a content policy violation error
@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping:
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
provider_specific_fields={
# Preserve legacy key for backward compatibility.
"innererror": inner_error,

View file

@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.prompt_templates.common_utils import (
content_parts,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
return any(_is_list_only_content_part(part) for part in content_parts(message))
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
content: Final = message.get("content")
if not isinstance(content, list):
return ()
try:
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
except ValidationError:
return ()
return tuple(part for part in parts if part.get("type") == "text")
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
return message
text_only: Final = {**message, "content": list(text_parts)}
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
return None
return PromptCacheBreakpoint(mode="explicit")
def _part_with_foundry_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _FIELDS.validate_python(part)
if "prompt_cache_breakpoint" not in fields:
return fields
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
parts: Final = content_parts(renamed) if drop_params else ()
if not parts:
return renamed
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
model_info: Final = litellm_params.get("model_info")
if not isinstance(model_info, Mapping):
return any(
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
)
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
if isinstance(deployment_flag, bool):
return deployment_flag
return any(
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
for name in named_models
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig):
self,
messages: list[AllMessageValues],
model: str,
reads_breakpoints: bool | None = None,
drop_params: bool = False,
) -> list:
"""
- Azure AI Studio doesn't support content as a list. This handles:
@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig):
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.
2. Transforms list content to a string.
2. Transforms list content to a string, except on deployments that read OpenAI explicit
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
content part can carry the breakpoint (a message-level one is silently ignored), so those
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
way the openai provider renames it. Every other part is dropped, exactly as the string
conversion drops it, before the marks are renamed, so a message-level mark lands on the
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
shape Foundry takes, and a malformed one is dropped.
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
deployment_reads_breakpoints: Final = (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
if reads_breakpoints is None
else reads_breakpoints
)
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
stripped_messages: Final = (
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
if deployment_reads_breakpoints
else copied_messages
)
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig):
if _has_list_only_content_part(message):
continue
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
message_dict["content"] = list(text_parts)
continue
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig):
if extra_body and isinstance(extra_body, dict):
optional_params.update(extra_body)
optional_params.pop("max_retries", None)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
return {
"model": model,
"messages": self._transform_messages(
messages,
model,
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
bool(litellm_params.get("drop_params") or litellm.drop_params),
),
**optional_params,
}
def transform_response(
self,

View file

@ -1,7 +1,12 @@
from typing import TYPE_CHECKING, Any, Final
import math
from collections.abc import Mapping
from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
import httpx
from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.image_generation.transformation import (
@ -18,6 +23,67 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float)
def _token_count_or_none(value: object) -> int | float | None:
if value is None:
return None
try:
count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value)
except ValidationError:
verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent")
return None
if not math.isfinite(count):
verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent")
return None
return count
_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)]
@with_config(ConfigDict(extra="allow"))
class _MAIImageUsageDetails(TypedDict, total=False):
text_tokens: ReadOnly[NotRequired[_TokenCount]]
image_tokens: ReadOnly[NotRequired[_TokenCount]]
@with_config(ConfigDict(extra="allow"))
class _MAIImageUsage(TypedDict, total=False):
input_tokens_details: ReadOnly[NotRequired[object]]
num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]]
num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]]
input_tokens: ReadOnly[NotRequired[_TokenCount]]
output_tokens: ReadOnly[NotRequired[_TokenCount]]
num_output_tokens: ReadOnly[NotRequired[_TokenCount]]
output_image_tokens: ReadOnly[NotRequired[_TokenCount]]
total_tokens: ReadOnly[NotRequired[_TokenCount]]
@with_config(ConfigDict(extra="allow"))
class _MAIImageResponse(TypedDict, total=False):
usage: ReadOnly[NotRequired[object]]
_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails)
_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage)
_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None:
if usage is None:
return None
try:
return _MAI_RAW_USAGE_ADAPTER.validate_python(usage)
except ValidationError:
verbose_logger.warning("MAI image usage is not an object, counting it as absent")
return None
_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse)
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
return "maiimage" in model_normalized
@staticmethod
def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]:
def normalize_mai_image_usage(usage: object) -> Mapping[str, object]:
"""Map Azure MAI usage fields to OpenAI ImageUsage schema."""
if usage is None:
raw_usage: Final = _raw_usage_or_none(usage)
if raw_usage is None:
return {
"input_tokens": 0,
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"total_tokens": 0,
}
normalized_usage: Final = dict(usage)
input_tokens_details = normalized_usage.get("input_tokens_details")
if not isinstance(input_tokens_details, dict):
input_tokens_details = {}
normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage)
input_tokens_details_value: Final = normalized_usage.get("input_tokens_details")
input_tokens_details: Final[_MAIImageUsageDetails] = (
_MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value)
if isinstance(input_tokens_details_value, dict)
else {}
)
text_tokens = normalized_usage.get("num_input_text_tokens")
if text_tokens is None:
@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
if total_tokens is None:
total_tokens = input_tokens + output_tokens
normalized_usage.update(
{
"input_tokens": input_tokens,
"input_tokens_details": {
"image_tokens": image_tokens,
"text_tokens": text_tokens,
},
"output_tokens": output_tokens,
"total_tokens": total_tokens,
}
)
return normalized_usage
return {
**raw_usage,
"input_tokens": input_tokens,
"input_tokens_details": {
"image_tokens": image_tokens,
"text_tokens": text_tokens,
},
"output_tokens": output_tokens,
"total_tokens": total_tokens,
}
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
return ["n", "size"]
@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
json_mode: bool | None = None,
) -> ImageResponse:
try:
response: Final = raw_response.json()
response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json())
except Exception:
raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
if "usage" in response:
response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
response: Final = (
{
**response_json,
"usage": self.normalize_mai_image_usage(response_json["usage"]),
}
if "usage" in response_json
else dict(response_json)
)
logging_obj.post_call(
input=request_data.get("prompt", ""),

View file

@ -1,10 +1,12 @@
import dataclasses
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from types import TracebackType
from typing import TYPE_CHECKING, Any, Protocol
from typing import TYPE_CHECKING, Any, Literal, Protocol
import httpx
from litellm._uuid import uuid
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
@ -22,6 +24,26 @@ else:
LiteLLMLoggingObj = Any
@dataclasses.dataclass(frozen=True, slots=True)
class RealtimeSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
message: str
param: str
def error_event(self, client_event_id: object) -> dict[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": self.code,
"message": self.message,
"param": self.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
class RealtimeBackend(Protocol):
async def __aenter__(self) -> object: ...
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def builds_setup_from_session_update(self) -> bool:
return False
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None

View file

@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping
from typing import TYPE_CHECKING, Final, get_args, overload
import httpx
from pydantic import TypeAdapter
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config
from .cohere_transformation import BedrockCohereEmbeddingConfig
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM):
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
return response.json()
return _JSON_OBJECT.validate_python(response.json())
async def _make_async_call(
self,
@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM):
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
return response.json()
return _JSON_OBJECT.validate_python(response.json())
def _transform_response(
self,
@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM):
# Parse response
if response.status_code == 200:
return response.json()
return _JSON_OBJECT.validate_python(response.json())
else:
raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")

View file

@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import redact_string, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
from .transformation import BedrockRealtimeConfig, requested_modalities
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -256,23 +256,11 @@ async def _ack_session_update(
async def _reject_session_update(
client_ws: RealtimeClientWebSocket,
parsed_client_message: Mapping[str, JsonValue],
rejection: BedrockSessionUpdateRejection,
rejection: RealtimeSessionUpdateRejection,
) -> bool:
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
client_event_id: Final = parsed_client_message.get("event_id")
error_event: Final = {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": rejection.code,
"message": rejection.message,
"param": rejection.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
try:
await client_ws.send_text(json.dumps(error_event))
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
return False

View file

@ -11,7 +11,7 @@ import math
import uuid as uuid_lib
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal
from typing import Final
import httpx
from pydantic import ConfigDict, JsonValue, TypeAdapter
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.base import LiteLLMBaseModel
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockLockedSessionSettings:
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
max_tokens=max_tokens,
)
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
"""The error for a session.update this session cannot apply, or None when it applies."""
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if not self.prompt_started:
return None
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
param="session",

View file

@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
def custom_llm_provider(self) -> str:
return "bigmodel"
@property
def prompts_unsupported_json_schema(self) -> bool:
return False
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)

View file

@ -0,0 +1,95 @@
from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.secret_managers.main import get_secret_str
SUPPORTED_OPENAI_PARAMS: Final = (
"stream",
"stream_options",
"frequency_penalty",
"presence_penalty",
"logit_bias",
"logprobs",
"top_logprobs",
"max_tokens",
"max_completion_tokens",
"n",
"stop",
"temperature",
"top_p",
"seed",
"response_format",
"tools",
"tool_choice",
"parallel_tool_calls",
"user",
"store",
"service_tier",
)
class ClfAiGatewayConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> str | None:
return "clf_ai_gateway"
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return
supports_reasoning: Final = litellm.supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
extra: Final = ("reasoning_effort",) if supports_reasoning else ()
return [*SUPPORTED_OPENAI_PARAMS, *extra]
def map_openai_params(
self,
non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature
requested: Final = non_default_params.get("reasoning_effort")
requested_mapping: Final = as_str_mapping(requested)
effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested
declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway")
drop_enabled: Final = drop_params or litellm.drop_params is True
if requested_mapping is not None and effort is None and not drop_enabled:
raise UnsupportedParamsError(
message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop "
"unsupported openai params from the call, set `litellm.drop_params = True`",
llm_provider="clf_ai_gateway",
model=model,
status_code=400,
)
drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared
if drops_effort:
verbose_logger.warning(
"Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.",
effort,
model,
declared,
)
others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"}
normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others
forwarded: Final = others if drops_effort else normalized
mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict
non_default_params=forwarded,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)
return mapped
def get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
resolved_api_base: Final = (
api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1"
)
dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY")
return resolved_api_base, dynamic_api_key

View file

@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
),
)
# Auto-send session setup if the provider requires it (e.g.
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
# Auto-send session setup at connect unless the provider builds it from the
# client's first session.update (Gemini/Vertex AI Live accept exactly one
# setup, so RealTimeStreaming sends theirs on the first client message).
# Build the streaming handler first so a transcription guardrail's
# auto-response disable can be folded into this one setup: Gemini
# rejects a second setup, so a follow-up disable would be dropped
# and the guardrail bypassed.
_session_config: str | None = None
if provider_config.requires_session_configuration():
sends_setup_at_connect: Final = (
provider_config.requires_session_configuration()
and not provider_config.builds_setup_from_session_update()
)
if sends_setup_at_connect:
_session_config = provider_config.session_configuration_request(model)
if _session_config:
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(

View file

@ -9,6 +9,7 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object
from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data
"""LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a
@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
if "application/json" not in raw_response.headers.get("content-type", ""):
return TranscriptionResponse(text=raw_response.text)
body: Final = raw_response.json()
body: Final = _JSON_OBJECT.validate_python(raw_response.json())
response: Final[TranscriptionResponse] = convert_to_model_response_object(
response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription"
)

View file

@ -7,6 +7,7 @@ from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, Required, TypedDict
import litellm
@ -15,8 +16,9 @@ from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
{
"achernar",
"achird",
"algenib",
"algieba",
"alnilam",
"aoede",
"autonoe",
"callirrhoe",
"charon",
"despina",
"enceladus",
"erinome",
"fenrir",
"gacrux",
"iapetus",
"kore",
"laomedeia",
"leda",
"orus",
"puck",
"pulcherrima",
"rasalgethi",
"sadachbia",
"sadaltager",
"schedar",
"sulafat",
"umbriel",
"vindemiatrix",
"zephyr",
"zubenelgenubi",
}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
OpenAI stock voice names have no Gemini equivalent, so they are always
dropped, and so is any name outside Google's documented Live voices when
``drop_undocumented`` is set, keeping the model's default voice. Every
other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
if not isinstance(voice, str):
return VertexGeminiConfig().map_audio_params({"voice": voice})
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
if drop_undocumented and _is_undocumented_live_voice(voice):
verbose_logger.warning(
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig().map_audio_params({"voice": voice})
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
return max(len(data) * 3 // 4 - padding, 0)
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
def _chat_completion_tool(tool: object) -> dict[str, object]:
fields: Final = as_str_mapping(tool) or {}
if fields.get("type") != "function" or "function" in fields:
return {**fields}
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
parts: Final = system_instruction.get("parts", []) if system_instruction else []
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
return "\n".join(texts) if texts else None
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
def __init__(self):
def __init__(self, drop_params: bool = False):
super().__init__()
self.drop_params = drop_params
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
self._manual_activity_open: bool = False
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
return True
def _rejects_undocumented_voices(self, model: str) -> bool:
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
detail_mapping: Final = as_str_mapping(details)
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
except (json.JSONDecodeError, TypeError, AttributeError):
return False
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
return ()
self._manual_activity_open = True
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
return (json.dumps({"realtimeInput": activity_start}),)
def _end_manual_activity(self) -> tuple[str, ...]:
if not self._manual_activity_open:
return ()
self._manual_activity_open = False
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
return (json.dumps({"realtimeInput": activity_end}),)
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
if self._manual_turn_detection_enabled(session_configuration_request):
realtime_input_dict: BidiGenerateContentRealtimeInput = {
"activityEnd": True,
}
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
else:
realtime_input_dict = {"audioStreamEnd": True}
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
return [json.dumps({"realtimeInput": realtime_input_dict})]
return list(self._end_manual_activity())
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
return [json.dumps({"realtimeInput": audio_stream_end})]
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"voice",
]
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
if "generationConfig" not in optional_params:
optional_params["generationConfig"] = {}
for key, value in non_default_params.items():
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
elif key == "temperature":
optional_params["generationConfig"]["temperature"] = value
elif key == "max_response_output_tokens":
elif key == "max_response_output_tokens" and isinstance(value, int):
optional_params["generationConfig"]["maxOutputTokens"] = value
elif key == "modalities":
optional_params["generationConfig"]["responseModalities"] = [
modality.upper() for modality in cast(list[str], value)
]
optional_params["generationConfig"]["responseModalities"] = list(
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
)
elif key == "tools":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
vertex_gemini_config = VertexGeminiConfig()
# Tools should be at the top level of setup, not inside generationConfig
optional_params["tools"] = vertex_gemini_config.map_function(
value=value, optional_params=optional_params
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
optional_params=optional_params,
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
continue
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
if transformed_audio_activity_config:
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "turn_detection":
realtime_input_config = self._realtime_input_config(cast(object, value))
if realtime_input_config is not None:
optional_params["realtimeInputConfig"] = realtime_input_config
elif key == "voice":
speech_config = _gemini_live_speech_config(value)
speech_config = _gemini_live_speech_config(
value, self.drop_params and self._rejects_undocumented_voices(model)
)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
optional_params.pop("generationConfig")
return optional_params
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
if turn_detection is None:
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
return None
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
if not transformed_audio_activity_config:
return None
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
@staticmethod
def _extract_turn_detection(session: dict) -> dict | None:
"""Extract turn_detection from a session.update payload.
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(input_cfg, dict):
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
normalized["input_audio_transcription"] = input_cfg["transcription"]
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
normalized["turn_detection"] = input_cfg["turn_detection"]
output_cfg: Final = audio.get("output")
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
normalized["voice"] = output_cfg["voice"]
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return normalized
@staticmethod
def _model_cost_entry(model: str) -> dict:
def _model_cost_entry(model: str) -> Mapping[str, object]:
entry = litellm.model_cost.get(model)
if entry is None:
stripped: Final = model.split("/", 1)[-1]
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
def _requested_response_modalities(value: object) -> tuple[str, ...]:
if isinstance(value, str):
return (value.upper(),)
if not isinstance(value, (list, tuple)):
return ()
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
@staticmethod
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
if unknown:
return unknown
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
return ("TEXT",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
return ("AUDIO",)
return ()
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
if self.drop_params:
return None
if setup_sent:
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message=(
"Gemini Live accepts one session configuration per connection, and this session already sent "
"its configuration, so this session.update was not applied. Put every setting in session.update "
"before the first conversation content, or set drop_params to drop it instead"
),
param="session",
)
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
@staticmethod
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
)
if not unsupported:
return None
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.{param}': {model} cannot produce "
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
),
param=f"session.{param}",
)
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
output_voice: Final = audio_output.get("voice")
voice: Final = output_voice or session.get("voice")
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
return None
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
"Set drop_params to drop it instead"
),
param=param,
)
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
text-only ones such as transcribe-live), then keep a single modality
because Gemini Live rejects a setup naming two."""
return GeminiRealtimeConfig._single_response_modality(
GeminiRealtimeConfig._swap_unsupported_response_modalities(
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
)
)
@staticmethod
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
normalized: Final = tuple(str(modality).upper() for modality in modalities)
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
if len(known) < len(normalized):
verbose_logger.warning(
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
model,
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
)
return known or (GeminiRealtimeConfig._default_response_modality(model),)
@staticmethod
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
verbose_logger.warning(
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
model,
)
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
if len(modalities) <= 1:
return modalities
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
verbose_logger.warning(
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
chosen,
", ".join(modalities),
)
return (chosen,)
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
generation_config: Final = setup.get("generationConfig")
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_payload = self._normalize_session_payload_for_mapping(session_payload)
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
new_overrides: Final = self.map_openai_params(
optional_params={}, non_default_params=session_payload, model=model
)
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
message_str = str(message)
raise ValueError(f"Invalid JSON message: {message_str}")
messages: Final[list[str]] = []
msg_type: Final = json_message.get("type")
if msg_type == "session.update":
return self._handle_session_update(json_message, model, session_configuration_request)
if msg_type == "response.create":
return [] # Gemini responds automatically; nothing to forward
return list(self._end_manual_activity())
if msg_type == "conversation.item.create":
return self._handle_conversation_item(json_message)
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
messages.append(gemini_msg)
return messages
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
logging_session_id: str,
session_configuration_request: str | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
session_configuration_request_dict: Final = _parse_setup(
session_configuration_request or self.session_configuration_request(model)
)
_model: Final = session_configuration_request_dict.get("model") or model
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
session: Final = OpenAIRealtimeStreamSession(
id=logging_session_id,
modalities=_modalities,
)
if _system_instruction is not None and isinstance(_system_instruction, str):
session["instructions"] = _system_instruction
if instructions is not None:
session["instructions"] = instructions
if _model is not None and isinstance(_model, str):
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
if "/models/" in _model:
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
return not litellm.gemini_live_defer_setup
def builds_setup_from_session_update(self) -> bool:
return True
def session_configuration_request(self, model: str) -> str:
"""

View file

@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from itertools import accumulate, count
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import (
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.mistral import (
MistralCacheKeyParams,
MistralFilePart,
MistralReasoningEffortParams,
MistralRequestRouting,
MistralTextBlock,
MistralThinkingBlock,
MistralToolCallMessage,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
requested: Final = as_str_mapping(reasoning_effort)
if requested is None:
return reasoning_effort, None
return requested.get("effort"), requested.get("summary")
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig):
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
"""
_is_base_class = False
temperature: int | None = None
top_p: int | None = None
max_tokens: int | None = None
@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
is_magistral: Final = "magistral" in model.lower()
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
return [
"stream",
"temperature",
@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig):
"parallel_tool_calls",
*(("thinking",) if is_magistral else ()),
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
]
def _map_tool_choice(self, tool_choice: str) -> str:
@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig):
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
optional_params["response_format"] = value
if param == "reasoning_effort" and "magistral" not in model.lower():
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
if (
param == "reasoning_effort"
and "magistral" not in model.lower()
and (
reasoning_level := self._reasoning_level_to_send(
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
)
)
is not None
):
optional_params["reasoning_effort"] = reasoning_level
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
if param == "parallel_tool_calls":
optional_params["parallel_tool_calls"] = value
if param in ("parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
return sendable
raise UnsupportedParamsError(
message=(
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=self.custom_llm_provider,
model=model,
)
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
hostname: Final = urlparse(resolved_api_base).hostname or ""
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
@staticmethod
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
if not self.accepts_prompt_cache_key_at(api_base):
return "since only the Mistral API (*.mistral.ai) takes it"
if not self.takes_prompt_cache_key_value(prompt_cache_key):
return "since the Mistral API takes it only as a string of ASCII characters"
return None
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
requested_api_base: Final = routing.get("api_base")
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
allowed_openai_params: Final = routing.get("allowed_openai_params")
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
return False
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
if rejection is None:
return False
if litellm.drop_params is True or routing.get("drop_params") is True:
return True
requested_provider: Final = routing.get("custom_llm_provider")
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
raise UnsupportedParamsError(
message=(
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=provider,
model=model,
)
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
api_base = (
@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig):
messages = self._handle_message_with_file(messages)
return messages
@staticmethod
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
file_part: Final = block.get("file") if block.get("type") == "file" else None
file_id: Final = (
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
)
if not file_id:
return block
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
@classmethod
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
content: Final = message.get("content")
if not isinstance(content, list):
return message
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
)
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
"""
Mistral API supports only 'file_id' in message content with type 'file'.
"""
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
if any(c.get("type") == "file" for c in _content_block):
# If file content is present, we get file_id from 'file' attribute of content block
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
file_contents = [c for c in _content_block if c.get("type") == "file"]
for file_content in file_contents:
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
file_content.pop("file", None)
return messages
return [self._with_mistral_file_ids(m) for m in messages]
def _add_reasoning_system_prompt_if_needed(
self, messages: list[AllMessageValues], optional_params: dict
@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig):
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
withheld_params: Final = (
("client_metadata", "prompt_cache_key")
if "prompt_cache_key" in cache_key_params
and self._prompt_cache_key_withheld(
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
)
else ("client_metadata",)
)
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
# Call parent transform_request which handles _transform_messages
return super().transform_request(

View file

@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
responses_so_far: Sequence["ModelResponseStream"],
litellm_logging_obj: "LiteLLMLoggingObj | None",
) -> "ModelResponse":
"""``stream_chunk_builder`` folds every choice of a stream into one index-0
choice, so the stream is rebuilt one choice index at a time (every chunk
kept, its choices narrowed to that index, so usage-only chunks still
count) and the rebuilt choices are stitched into one response, each
carrying the index the stream gave it."""
choice_indices: Final = tuple(
sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
)

Some files were not shown because too many files have changed in this diff Show more