Merge remote-tracking branch 'origin/main' into litellm_responses_drop_malformed_values

# Conflicts:
#	litellm/responses/utils.py
This commit is contained in:
mateo-berri 2026-10-30 05:16:38 -07:00
commit 73d5ced9c5
258 changed files with 16864 additions and 4496 deletions

View file

@ -1,152 +0,0 @@
name: Lens installation smoke
on:
workflow_dispatch:
inputs:
lens_image:
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
required: true
type: string
lens_upgrade_image:
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
required: true
type: string
gateway_baseline_ref:
description: Previously qualified gateway source commit, full SHA
required: true
type: string
permissions:
contents: read
concurrency:
group: lens-install-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-images:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- component: gateway
dockerfile: gateway/Dockerfile
- component: backend
dockerfile: backend/Dockerfile
- component: ui
dockerfile: ui/Dockerfile
- component: migrations
dockerfile: migrations/Dockerfile
- component: monolith
dockerfile: Dockerfile
- component: gateway
dockerfile: gateway/Dockerfile
baseline: true
- component: backend
dockerfile: backend/Dockerfile
baseline: true
- component: monolith
dockerfile: Dockerfile
baseline: true
env:
COMPONENT: ${{ matrix.component }}
DOCKERFILE: ${{ matrix.dockerfile }}
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
CURRENT_REF: ${{ github.sha }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ env.SOURCE_REF }}
- name: Verify the selected source
shell: bash -euo pipefail {0}
run: |
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
test "$SOURCE_REF" != "$CURRENT_REF"
fi
- name: Build the matching release image
run: |
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
- name: Save the matching release image
run: |
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
compression-level: 0
retention-days: 3
if-no-files-found: error
overwrite: true
helm-install:
needs: build-images
runs-on: ubuntu-latest-16-cores
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-install-*-${{ github.sha }}
merge-multiple: true
path: ${{ runner.temp }}/lens-install-images
- name: Load the matching release images
run: |
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
gzip -dc "$archive" | docker load
rm "$archive"
done
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
with:
cosign-release: v2.4.3
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ vars.LENS_READ_USER || github.actor }}
password: ${{ secrets.LENS_READ_TOKEN }}
- name: Verify independent Lens images before loading them
env:
LENS_IMAGE: ${{ inputs.lens_image }}
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
shell: bash -euo pipefail {0}
run: |
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
docker pull "$image"
done
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
- name: Install pinned Kubernetes test tools
run: |
curl --fail --location --output "$RUNNER_TEMP/kind" \
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
chmod +x "$RUNNER_TEMP/kind"
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
chmod +x "$RUNNER_TEMP/kubectl"
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
- name: Install, ingest, upgrade, and restart both charts
run: bash tests/e2e/migrations/lens_helm_smoke.sh

View file

@ -227,6 +227,7 @@ jobs:
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
tests/unit/llms/test_predibase_transformation.py
tests/unit/llms/test_subscription_accounts.py
tests/unit/llms/a2a
tests/unit/llms/aiml
tests/unit/llms/aiohttp_openai

View file

@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/subscription_accounts/",
"/cost_optimization/",
"/cost/",
"/credentials",

View file

@ -5,8 +5,5 @@ dependencies:
- name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: 18.19.1
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
generated: '2026-10-09T04:18:10.643228000Z'
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
generated: '2026-10-30T02:13:11.505726000Z'

View file

@ -39,6 +39,3 @@ dependencies:
version: "18.19.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -256,9 +256,7 @@ itself.
## Connect Lens
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
## Shutdown accounting

View file

@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -402,95 +398,28 @@ shutdown drain window.
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -1,8 +1,6 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "litellm.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- $lensMode := include "litellm.lens.mode" . -}}
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@ -46,18 +44,18 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
name: {{ $.Values.lens.service.name }}
port:
number: {{ $.Values.lensWorker.service.port }}
number: {{ $.Values.lens.service.port }}
{{- else }}
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
servicePort: {{ $.Values.lensWorker.service.port }}
serviceName: {{ $.Values.lens.service.name }}
servicePort: {{ $.Values.lens.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -0,0 +1,128 @@
suite: lens connection
templates:
- deployment.yaml
- configmap-litellm.yaml
- ingress.yaml
tests:
- it: connects the proxy to an external Lens
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service ahead of the host paths
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- equal:
path: spec.rules[0].http.paths[0].path
value: /lens-ingest
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,84 +0,0 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

View file

@ -1,187 +0,0 @@
---
suite: Lens deployment modes
templates:
- ingress.yaml
- configmap-litellm.yaml
- deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,52 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- configmap-litellm.yaml
- deployment.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,59 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

View file

@ -1,167 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- configmap-litellm.yaml
- deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects deployment.yaml to the shared Lens service
template: deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.hosts:
- host: gateway.example
paths:
- path: /
pathType: Prefix
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference

View file

@ -1,162 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

View file

@ -668,70 +668,15 @@ serviceMonitor:
# - test-namespace
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -1,6 +0,0 @@
dependencies:
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
generated: '2026-10-09T04:18:10.642706000Z'

View file

@ -6,8 +6,3 @@ version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
dependencies:
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -1,10 +1,8 @@
# LiteLLM componentized chart
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
This chart deploys separate gateway, backend and UI services
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.

View file

@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
collector containers through an emptyDir. Empty when the sidecar is off
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
*/}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -635,93 +631,27 @@ shutdown drain window.
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -166,14 +166,14 @@ spec:
- host: {{ $componentHosts.gateway | quote }}
http:
paths:
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
- path: /
pathType: Prefix
@ -249,15 +249,15 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -15,7 +15,6 @@ tests:
ingress.componentHosts.ui: ui.example.com
ingress.componentHosts.backend: backend.example.com
ingress.componentHosts.gateway: gateway.example.com
lensWorker.mode: disabled
ingress.tls:
- hosts:
- ui.example.com
@ -83,12 +82,12 @@ tests:
ui.backendUrl: https://backend.example.com
ui.publicUrl: https://ui.example.com
backend.gatewayUrl: https://gateway.example.com
lensWorker.mode: external
lensWorker.externalServiceName: existing-lens
lensWorker.externalUrl: https://lens.example.com
lensWorker.publicUrl: https://lens.example.com
lensWorker.serviceTokenSecret.name: existing-lens-signing
lensWorker.gateway.secretName: existing-lens-signing
lens.enabled: true
lens.service.name: existing-lens
lens.url: http://existing-lens:4318
lens.publicUrl: https://gateway.example.com/lens-ingest
lens.serviceTokenSecret.name: existing-lens-signing
lens.gatewaySecret.name: existing-lens-signing
asserts:
- equal:
path: spec.rules[2].http.paths[0].path

View file

@ -0,0 +1,166 @@
suite: lens connection
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
- gateway/configmap.yaml
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: connects the gateway to an external Lens
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: connects the backend to an external Lens
template: backend/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service on the single ingress host
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,86 +0,0 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

View file

@ -1,208 +0,0 @@
---
suite: Lens deployment modes
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
- ingress.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: keeps the ingestion endpoint on an independently deployed Lens service
template: ingress.yaml
set:
ingress.enabled: true
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: existing-lens
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,54 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: backend/deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,61 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

View file

@ -1,196 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- gateway/configmap.yaml
- gateway/deployment.yaml
- backend/deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects gateway/deployment.yaml to the shared Lens service
template: gateway/deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: connects backend/deployment.yaml to the shared Lens service
template: backend/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.host: gateway.example
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: gateway/deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: omits Lens connection settings when disabled in backend/deployment.yaml
template: backend/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: gateway/deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference
values:
- ./values/required.yaml

View file

@ -1,164 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

View file

@ -1,79 +0,0 @@
suite: Independent Lens release and credentials
templates:
- lens/deployment.yaml
values:
- ./values/required.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
tests:
- it: selects a Lens release independently of the gateway
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
lensWorker.image.tag: 2.3.4
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:2.3.4
- it: retains the pinned Lens chart default when only the gateway changes
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:0.1.0-dev.0
- it: selects development publications explicitly
set:
lensWorker.image.repository: ghcr.io/berriai/lens-dev
lensWorker.image.tag: sha-0123456789abcdef
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
- it: preserves a verified registry mirror and digest
set:
lensWorker.image.repository: registry.example/lens
lensWorker.image.tag: replaced-label
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- it: rejects a malformed digest
set:
lensWorker.image.digest: sha256:invalid
asserts:
- failedTemplate:
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
- it: keeps the legacy enablement switch working
set:
lensWorker.enabled: false
asserts:
- hasDocuments: {count: 0}
- it: keeps isolated local analysis with delegated gateway access
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: LENS_MODE, value: standalone}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- equal:
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
value: true
- equal:
path: spec.template.spec.volumes[0].emptyDir
value: {medium: Memory, sizeLimit: 1Gi}

View file

@ -654,70 +654,15 @@ ui:
topologySpreadConstraints: []
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -0,0 +1,21 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
"subscription_account_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"account_id" TEXT NOT NULL,
"label" TEXT,
"monthly_fee" DOUBLE PRECISION NOT NULL,
"currency" TEXT NOT NULL,
"billing_period_start" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

View file

@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
baseten_models.discard(key)

View file

@ -7,6 +7,7 @@ from typing import Final, Literal
from litellm.litellm_core_utils.env_utils import (
get_env_float,
get_env_float_in_range,
get_env_float_or_none,
get_env_int,
get_env_int_in_range,
@ -67,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"retry_after",
"fallbacks",
"context_window_fallbacks",
"content_policy_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
@ -149,6 +151,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
"DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
) # default cooldown a deployment if 50% of requests fail in a given minute
DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
)
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
@ -416,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@ -893,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
}
)
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
@ -1270,7 +1278,7 @@ empower_models: Final = set(
]
)
together_ai_models: Final[set] = set(
together_ai_models: Final[set[str]] = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",

View file

@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost(
return litellm_logging_obj.get_response_ms()
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
def _adjusted_provider_billed_cost(
reported_cost: float,
custom_llm_provider: str | None,
@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost(
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=None,
completion_tokens_cost_usd_dollar=None,
cost_for_built_in_tools_cost_usd_dollar=None,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=adjusted_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
discount_percent=discount_percent,
@ -2376,6 +2382,15 @@ def response_cost_calculator(
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=provider_response_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=provider_response_cost,
original_cost=provider_response_cost,
)
return provider_response_cost
response_cost = completion_cost(

View file

@ -9,6 +9,7 @@
## LiteLLM versions of the OpenAI Exception Types
import datetime
import enum
from collections.abc import Mapping, Sequence
from typing import Any, Final
@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
return None
def _upstream_body(response: object) -> bytes:
if not isinstance(response, httpx.Response):
return b""
try:
return response.content
except httpx.ResponseNotRead:
return b""
def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
if not isinstance(upstream, httpx.Response):
return httpx.Request(method="POST", url=fallback_url)
try:
upstream_request: Final = upstream.request
except RuntimeError:
return httpx.Request(method="POST", url=fallback_url)
return httpx.Request(
method=upstream_request.method,
url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
)
def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
if not isinstance(upstream, httpx.Response):
return None
try:
return upstream.elapsed
except RuntimeError:
return None
def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
response: Final = httpx.Response(
status_code=status_code,
stream=httpx.ByteStream(_upstream_body(upstream)),
request=_request_without_credentials(upstream, fallback_url),
)
_ = response.read()
response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
elapsed: Final = _upstream_elapsed(upstream)
if elapsed is not None:
response.elapsed = elapsed
return response
_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
_MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
# headers stay reachable on `e.response.headers` for callers that
# explicitly want them; only the proxy-supplied `headers=` kwarg
# makes it onto `self.headers`.
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
# Mirrors FastAPI HTTPException.detail so the same instance can be
# serialized through both the ProxyException and HTTPException paths.
self.detail = detail if detail is not None else self.message
self.response = httpx.Response(
status_code=429,
headers=_response_headers,
content=response.content if response is not None else None,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message, response=self.response, body=None
) # Call the base class constructor with the parameters it needs
@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -908,6 +924,7 @@ class APIError(openai.APIError):
max_retries: int | None = None,
num_retries: int | None = None,
body: object | None = None,
response: httpx.Response | None = None,
):
self.status_code = status_code
self.message = f"litellm.APIError: {message}"
@ -916,6 +933,8 @@ class APIError(openai.APIError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
if response is not None:
self.response = response
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=body)

View file

@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
with_prompt_cache_breakpoint,
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
def _hosted_openai_dialect_flag(
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
"""
import litellm
exact_entry: Final = litellm.model_cost.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
return None
provider: Final = custom_llm_provider or resolve_provider(model)
if provider is None or provider == "openai":
return None
if isinstance(exact_entry, dict) and exact_provider == provider:
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
from litellm.utils import get_potential_model_names
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)

View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
)
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
if not isinstance(metadata, dict):
return []
return metadata.get("guardrails") or []
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
return True
raise
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
"""
Returns the guardrail(s) to be run from the metadata or root
Returns the guardrail(s) to be run from the root, else from both metadata fields together
"""
if "guardrails" in data:
return data["guardrails"]
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
# writes to "metadata"; for thread/assistant endpoints it writes to
# "litellm_metadata". We check the one that actually contains the "guardrails"
# key so that a non-empty litellm_metadata without guardrails does not shadow
# the merged list stored in metadata (which would cause team guardrails to be
# silently skipped while default_on=True policy guardrails still fire).
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key) or {}
if isinstance(meta, dict) and "guardrails" in meta:
return meta.get("guardrails") or []
return []
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
return [
*_guardrails_listed_in(request_fields.get("metadata")),
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
]
def _guardrail_is_in_requested_guardrails(
self,
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
) -> bool:
for _guardrail in requested_guardrails:
if isinstance(_guardrail, dict):

View file

@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_LensReview",
"LiteLLM_LensIngestionKey",
"LiteLLM_LensWorker",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_LensSignalConfig",
"LiteLLM_LensTraceSignal",
)

View file

@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
emit_native_blocks: Final = native_tool is not None or (
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
)
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
content: Final[list[dict[str, object]]] = []
if native_tool is not None:
if emit_native_blocks:
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
tool_name: Final = native_tool.get("name") or "web_search"
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
content.append(
{
"type": "server_tool_use",
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
}
)
content.append(
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
WebSearchTransformation.build_web_search_outcome_block(
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
response: Final[dict[str, object]] = {
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
len(search_result_text),
native_tool is not None,
emit_native_blocks,
)
return response

View file

@ -412,6 +412,7 @@ class WebSearchTransformation:
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: SearchResponse | None,
with_snippets: bool = True,
) -> dict[str, object]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
@ -445,6 +446,8 @@ class WebSearchTransformation:
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
with_snippets: False when a text block in the same reply already
carries the snippets.
"""
items: Final[list[dict[str, object]]] = []
if search_response is not None:
@ -460,7 +463,7 @@ class WebSearchTransformation:
"title": title,
"page_age": page_age,
"encrypted_content": "",
"snippet": getattr(r, "snippet", "") or "",
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
}
)
return {
@ -481,12 +484,15 @@ class WebSearchTransformation:
}
@staticmethod
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
def build_web_search_outcome_block(
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
) -> dict[str, object]:
match outcome:
case SearchSucceeded(response=response):
return WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=response,
with_snippets=with_snippets,
)
case SearchFailed(error_code=error_code):
return WebSearchTransformation.build_web_search_tool_result_error_block(

View file

@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
return getattr(chunk, "usage", None)
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
def _numeric(value: object) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
reading litellm's bill back as if the upstream had reported it."""
hidden: Final[object] = getattr(response, "_hidden_params", None)
if isinstance(hidden, dict):
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
upstream: Final = (
hidden_params[UPSTREAM_USAGE_COST]
if UPSTREAM_USAGE_COST in hidden_params
else _numeric(getattr(usage, "cost", None))
)
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
setattr(usage, "cost", bill)
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
return None
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
cost: Final = (
_numeric(hidden[UPSTREAM_USAGE_COST])
if hidden is not None and UPSTREAM_USAGE_COST in hidden
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
)
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None

View file

@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
Values outside the range fall back to the default and warn, so a misconfigured knob can
neither crash the caller nor silently change the meaning of what it computes.
"""
value: Final = get_env_int(env_var, default)
return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
"""Float counterpart of `get_env_int_in_range`."""
return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
if minimum <= value <= maximum:
return value
logging.getLogger("LiteLLM").warning(

View file

@ -37,6 +37,7 @@ from ..exceptions import (
ServiceUnavailableError,
Timeout,
UnprocessableEntityError,
error_response_without_credentials,
)
@ -305,6 +306,16 @@ def _litellm_proxy_response(
)
def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
if not isinstance(response, httpx.Response):
return response
return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
return response if response is not None and (response.headers or response.content) else None
def _map_openai_exception(
*,
model: str,
@ -315,7 +326,7 @@ def _map_openai_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
raise ContentPolicyViolationError(
message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
@ -378,6 +389,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif (
("invalid_request_error" in error_str and "content_policy_violation" in error_str)
@ -449,6 +461,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
response=_upstream_reply(response),
)
elif "Request too large" in error_str:
raise RateLimitError(
@ -479,6 +492,7 @@ def _map_openai_exception(
model=model,
request=_request,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
@ -512,6 +526,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -563,6 +578,7 @@ def _map_openai_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -572,6 +588,7 @@ def _map_openai_exception(
model=model,
request=getattr(original_exception, "request", None),
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2019,6 +2036,7 @@ def _map_azure_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _without_credentials(getattr(original_exception, "response", None))
message = get_error_message(error_obj=original_exception)
if message is None:
if hasattr(original_exception, "message"):
@ -2054,7 +2072,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "DeploymentNotFound" in error_str:
raise NotFoundError(
@ -2062,7 +2080,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
error_str
@ -2076,6 +2094,7 @@ def _map_azure_exception(
model=model,
extra_information=extra_information,
original_exception=original_exception,
response=response,
)
elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
helpful_message: Final = (
@ -2094,7 +2113,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif getattr(original_exception, "status_code", None) in (None, 400) and (
@ -2106,7 +2125,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
@ -2115,7 +2134,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
@ -2124,7 +2143,7 @@ def _map_azure_exception(
llm_provider=custom_llm_provider,
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "Connection error" in error_str:
raise APIConnectionError(
@ -2140,7 +2159,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif original_exception.status_code == 401:
@ -2149,7 +2168,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 404:
raise NotFoundError(
@ -2157,7 +2176,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 408:
raise Timeout(
@ -2165,6 +2184,7 @@ def _map_azure_exception(
model=model,
litellm_debug_info=extra_information,
llm_provider="azure",
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -2172,7 +2192,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 429:
raise RateLimitError(
@ -2180,7 +2200,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 502:
raise BadGatewayError(
@ -2188,7 +2208,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 503:
raise ServiceUnavailableError(
@ -2196,7 +2216,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 504: # gateway timeout error
raise Timeout(
@ -2205,6 +2225,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
llm_provider="azure",
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -2214,6 +2235,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
model=model,
request=httpx.Request(method="POST", url="https://openai.com/"),
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2589,17 +2611,20 @@ def exception_type(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(
_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
),
)
if (
custom_llm_provider == "litellm_proxy"
): # handle special case where calling litellm proxy + exception str contains error message
extract_and_raise_litellm_exception(
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
error_str=error_str,
model=model,
custom_llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
body=getattr(mappable_exception, "body", None),
)
if (
custom_llm_provider == "openai"

View file

@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

View file

@ -3,6 +3,8 @@ Helper functions for health check calls.
"""
import base64
import itertools
import re
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
)
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
)
@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
return native(model, custom_llm_provider)
def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
try:
model_info: Final = ProviderConfigManager.get_provider_model_info(
model=None, provider=LlmProviders(custom_llm_provider)
)
return None if model_info is None else model_info.get_base_model(model)
except (ValueError, NotImplementedError):
return None
def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
if custom_llm_provider == "bedrock":
from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
return base_model is not None and not base_model.startswith(f"{segment}/")
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
provider_prefix: Final = f"{custom_llm_provider}/"
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
route_segments: Final = tuple(
itertools.takewhile(
lambda segment_index: _is_route_segment(
custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
),
range(len(segments) - 1),
)
)
route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
ranked_model_ids: Final = tuple(
(ranked_model.model.removeprefix(provider_prefix), ranked_model)
for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
)
served_model_ids: Final = sorted(
(ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
)
served_probes: Final = itertools.islice(
dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
)
provider_probes: Final = (
f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
)
return tuple(dict.fromkeys((*served_probes, *provider_probes)))
class HealthCheckHelpers:
@staticmethod
async def ahealth_check_wildcard_models(
@ -99,23 +160,16 @@ class HealthCheckHelpers:
litellm_logging_obj: "Logging",
) -> dict:
from litellm import acompletion
from litellm.litellm_core_utils.llm_request_utils import (
pick_cheapest_chat_models_from_llm_provider,
)
# this is a wildcard model, we need to pick a random model from the provider
cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
if len(cheapest_models) == 0:
probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
if len(probe_models) == 0:
raise Exception(
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
"no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
)
if len(cheapest_models) > 1:
fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list
else:
fallback_models = None
model_params["model"] = cheapest_models[0]
model_params["model"] = probe_models[0]
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["fallbacks"] = list(probe_models[1:]) or None
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
await acompletion(**model_params)
return {}

View file

@ -1,6 +1,11 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
@ -151,19 +156,52 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
Returns:
list[str]: A list of the n cheapest chat models.
"""
if custom_llm_provider not in litellm.models_by_provider:
return []
return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]]
@dataclass(frozen=True, slots=True)
class RankedChatModel:
model: str
route_rank: int
cost: float
is_past_deprecation_date: bool
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
ranked_rows: Final = (
(route_rank, cost, model)
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
)
for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider)
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
)
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost)))
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
class _CostMapDeprecation(TypedDict):
deprecation_date: NotRequired[ReadOnly[object]]
_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
)
except ValueError:
return False
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
for model in models:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
@ -171,7 +209,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
continue
if model_info.get("mode") != "chat":
continue
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
yield (
model,
(model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0),
_is_past_deprecation_date(model_info["key"]),
)
def _route_rank(model: str, custom_llm_provider: str) -> int | None:

View file

@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types

View file

@ -912,9 +912,9 @@ def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Create an Anthropic content block from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
@ -933,14 +933,7 @@ def create_anthropic_image_param(
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
@ -953,14 +946,7 @@ def create_anthropic_image_param(
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
_anthropic_image_param,
)
)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
@ -5878,38 +5869,36 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
Returns the prompt str that's passed to the model as a user message
"""
custom_prompt_details: dict | None = None
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict[
f"{model}/response_schema_prompt"
] # allow user to define custom response schema prompt by model
elif "response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
model_prompt_key: Final = f"{model}/response_schema_prompt"
custom_prompt_details: Final = prompts.get(
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
)
if custom_prompt_details is None:
return default_response_schema_prompt(response_schema=response_schema)
return render_custom_prompt_template(
template=custom_prompt_details,
messages=[{"role": "user", "content": f"{response_schema}"}],
model=model,
llm_provider=llm_provider,
)
def default_response_schema_prompt(response_schema: dict) -> str:
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
"""
Used if provider/model doesn't support 'response_schema' param.
This is the default prompt. Allow user to override this with a custom_prompt.
"""
prompt_str: Final = f"""Use this JSON schema:
```json
{response_schema}
```json
{json.dumps(response_schema, default=str)}
```"""
return prompt_str

View file

@ -1,5 +1,6 @@
import asyncio
import json
import time
import traceback
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.llms.base_llm.realtime.transformation import (
BaseRealtimeConfig,
RealtimeBackend,
RealtimeSessionUpdateRejection,
)
from litellm.types.llms.openai import (
OpenAIRealtimeEvents,
OpenAIRealtimeOutputItemDone,
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_MERGE_MAX_DEPTH: Final = 8
class _SessionUpdateFrame(TypedDict):
type: ReadOnly[str]
session: ReadOnly[Mapping[str, object]]
def _merge_session_fields(
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
) -> Mapping[str, object]:
def merged_value(key: str) -> object:
if key not in update:
return base[key]
value: Final = update[key]
base_value: Final = base.get(key)
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
return _merge_session_fields(
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
)
return update[key]
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
def _session_update_frame(session: Mapping[str, object]) -> str:
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
code="session_update_too_large",
message=(
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
"so this session.update was not applied"
),
param="session",
)
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": "audio_input_not_screenable",
"message": (
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
),
"param": None,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
def _decode_json_object(payload: str) -> Mapping[str, object]:
"""Decode a realtime frame into its top-level field mapping."""
return json.loads(payload)
@ -243,8 +309,17 @@ class RealTimeStreaming:
event_normalizer: RealtimeEventNormalizer | None = None,
usage_meter: RealtimeUsageMeter | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
clock: Callable[[], float] = time.monotonic,
):
self.websocket: _ClientWebSocket = websocket
self._clock: Final = clock
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
self._held_client_content: tuple[str, ...] = ()
self._held_client_content_bytes: int = 0
self._held_client_content_deadline: float | None = None
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
@ -300,6 +375,10 @@ class RealTimeStreaming:
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
# content (realtimeInput / clientContent / toolResponse) has been sent.
self._content_sent_after_setup: bool = False
self._held_session_update: Mapping[str, object] | None = None
self._applied_session_fields: Mapping[str, object] | None = None
self._held_session_update_bytes: int = 0
self._unscreened_audio_refused: bool = False
# Whether this is a transcription-only session (session.type == "transcription",
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
# their input_audio_transcription.completed usage drives duration-based cost.
@ -337,6 +416,17 @@ class RealTimeStreaming:
]
)
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
_CLIENT_CONTENT_TYPES = frozenset(
[
"conversation.item.create",
"response.create",
"input_audio_buffer.append",
"input_audio_buffer.commit",
"input_audio_buffer.end",
]
)
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@ -377,7 +467,7 @@ class RealTimeStreaming:
message_obj: dict[str, Any] = cast(dict[str, Any], message)
else:
message_obj = cast(dict[str, Any], json.loads(message))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
self._collect_tool_calls_from_response_done(message_obj)
if not self._should_store_message(message_obj):
return
try:
@ -502,7 +592,7 @@ class RealTimeStreaming:
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
if event_obj.get("type") != "response.done":
@ -607,6 +697,167 @@ class RealTimeStreaming:
self._usage_meter.observe_client_event(sent_event)
return True
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
return self.provider_config.builds_setup_from_session_update()
def _holds_session_updates_until_first_content(self) -> bool:
return self.session_configuration_request is None and self._builds_setup_lazily()
async def _rejected_setup_session_update(self, message: str) -> bool:
provider_config: Final = self.provider_config
if provider_config is None or not provider_config.builds_setup_from_session_update():
return False
client_event: Final = _decode_json_object(message)
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
if self._leaves_applied_setup_unchanged(update):
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
return True
rejection: Final = provider_config.setup_session_update_rejection(
self.model, update, setup_sent=self.session_configuration_request is not None
)
if rejection is None:
return False
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
return True
def _held_session_update_message(self) -> str:
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
async def _hold_session_update(self, message: str) -> None:
client_event: Final = _decode_json_object(message)
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
await self.websocket.send_text(
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
)
return
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
await self._acknowledge_session_setup(held)
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
if self.provider_config is None:
return None
preview: Final = self.provider_config.transform_realtime_request(
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
applied: Final = self._applied_session_fields
if self.session_configuration_request is None or applied is None:
return False
applied_setup: Final = self._setup_preview(applied)
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
if self.provider_config is None:
return
setup: Final = self._setup_preview(session)
if setup is None:
return
acknowledged: Final = self.provider_config.transform_session_created_event(
self.model, str(self.logging_obj.litellm_trace_id), setup
)
if acknowledged is None:
return
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
event_str: Final = json.dumps(event)
self.store_message(event_str)
await self._send_event_to_client(event, event_str)
async def _send_held_session_update_as_setup(self) -> None:
held: Final = self._held_session_update
if held is None:
return
message: Final = self._held_session_update_message()
self._held_session_update = None
self._held_session_update_bytes = 0
if await self._send_to_backend(message):
self._applied_session_fields = held
await self._send_held_client_content()
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
held_types: Final = (
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
if self._uses_deferred_backend_setup()
else RealTimeStreaming._HELD_CONTENT_TYPES
)
return (
msg_type in held_types
and self._held_session_update is not None
and self._holds_session_updates_until_first_content()
and not self._pending_messages_until_setup
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
)
def _hold_client_content(self, message: str) -> None:
if self._held_client_content_deadline is None:
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
self._held_client_content = (*self._held_client_content, message)
self._held_client_content_bytes += len(message.encode("utf-8"))
async def _send_held_client_content(self) -> None:
held: Final = self._held_client_content
self._held_client_content = ()
self._held_client_content_bytes = 0
self._held_client_content_deadline = None
for message in held:
await self._send_to_backend(message)
def _setup_deadline(self) -> float:
content_deadline: Final = self._held_client_content_deadline
return (
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
)
async def _send_setup_before_content(self, msg_type: str | None) -> None:
if self._holds_session_updates_until_first_content():
await self._send_held_session_update_as_setup()
await self._send_default_setup_before_first_client_message(msg_type)
async def _receive_client_message_before_setup_deadline(self) -> str | None:
if not self._holds_session_updates_until_first_content():
return await self._receive_client_message()
receive: Final = asyncio.ensure_future(self._receive_client_message())
try:
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
if not done:
await self._send_setup_before_content(None)
return await receive
finally:
receive.cancel()
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
if self.session_configuration_request is not None or self.provider_config is None:
return
if msg_type == "session.update" or (
self._uses_deferred_backend_setup()
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
):
return
default_setup: Final = self.provider_config.session_configuration_request(self.model)
if not default_setup:
return
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
await self.backend_ws.send(setup)
self._cache_session_configuration_request(setup)
self._applied_session_fields = _NO_SESSION_FIELDS
await self._send_held_client_content()
def _enforce_transcription_session_model(self, message: str) -> str:
"""Force client transcription session updates to the authorized model.
@ -851,6 +1102,20 @@ class RealTimeStreaming:
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
if pre_block_backend_message is None and self._is_transcription_session:
return
await self._send_setup_before_content("conversation.item.create")
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
@ -958,6 +1223,21 @@ class RealTimeStreaming:
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
return (
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
and self._builds_setup_lazily()
and self._should_disable_vad_auto_response()
)
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
return
self._unscreened_audio_refused = True
await self.websocket.send_text(
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
)
async def run_realtime_guardrails(
self,
transcript: str,
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
break
finally:
self._flushing_pending_messages_until_setup = False
if self.session_created_sent_to_client and self._builds_setup_lazily():
await self._maybe_send_guardrail_turn_detection_update()
continue
if self.session_created_sent_to_client:
# A synthetic session.created (with placeholder defaults) was
# already forwarded to the client when we connected. The
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
return (
close.code != 1000
and not self._backend_sent_frames
and not self._holds_session_updates_until_first_content()
)
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
client_event: _ClientEventFrame
try:
while True:
message = await self._receive_client_message()
message = await self._receive_client_message_before_setup_deadline()
if message is None:
return ClientLoopExit.CLIENT_DISCONNECTED
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
if (
msg_type == "session.update"
and self.session_configuration_request is None
and self._held_session_update is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
and (
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
except (json.JSONDecodeError, AttributeError):
pass
if self._refuses_unscreened_audio(msg_type):
await self._refuse_unscreened_audio(msg_type, message)
continue
## LOGGING
# Log after any in-place modifications (GA remap, guardrail
# turn_detection injection) so audit logs reflect what we
# actually forward to the backend.
self.store_input(message=message)
if self._holds_client_content(msg_type, message):
self._hold_client_content(message)
continue
if self._held_client_content and msg_type != "session.update":
await self._send_setup_before_content(msg_type)
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
await self._send_setup_before_content(msg_type)
if self._should_buffer_client_message_until_setup(message):
self._buffer_pending_message_until_setup(message)
continue
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
continue
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
await self._hold_session_update(message)
if self._pending_messages_until_setup:
await self._send_held_session_update_as_setup()
continue
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
await self._send_setup_before_content(msg_type)
try:
sent = await self._send_to_backend(message)
except RealtimeTranscriptionProtocolError as e:

View file

@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import (
USAGE_COST_BILLED_PROVIDERS,
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
chunk_usage,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
@ -1380,9 +1381,6 @@ def stream_chunk_builder_text_completion(
return TextCompletionResponse(**response)
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return None
@ -1390,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
return provider
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
@ -1404,9 +1398,6 @@ def _stream_builder_response_cost(
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
@ -1453,13 +1444,15 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
return
setattr(usage, "cost", computed_cost)
_set_hidden_response_cost(response, float(computed_cost))
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
def _field_of(value: object, name: str) -> object:

View file

@ -17,6 +17,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
AnthropicSearchResultParam,
CompactionBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -937,6 +939,8 @@ def _count_messages(
default_token_count,
params.pdf_pricing,
)
elif key == "provider_specific_fields":
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
@ -951,6 +955,25 @@ def _count_messages(
return num_tokens
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
fields: Final = as_str_mapping(provider_specific_fields)
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
block_fields: Final = (
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
)
return tuple(
content
for block in block_fields
if block is not None
and block.get("type") == "compaction"
and isinstance(content := block.get("content"), str)
and content
)
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
@ -1648,6 +1671,11 @@ def _count_anthropic_content(
return tokens
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1659,6 +1687,7 @@ def _count_content_list(
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| AnthropicSearchResultParam
| CompactionBlock
],
use_default_image_token_count: bool,
default_token_count: int | None,
@ -1670,8 +1699,10 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
if block_text:
num_tokens += count_function(block_text)
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1704,12 +1735,6 @@ def _count_content_list(
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -1727,7 +1752,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"compaction, tool_reference, search_result)."
)
return num_tokens
except Exception as e:

View file

@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
unappliable_request_rewrite,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
AllAnthropicToolsValues,
AnthropicMessagesRequest,
)
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
text: str
@runtime_checkable
class _WritableMessage(Protocol):
@overload
def get(self, key: str, /) -> object | None: ...
@ -195,6 +197,140 @@ class _ToolCallShape:
arguments: str
@dataclass(frozen=True, slots=True)
class _WebSearchString:
block_idx: int
result_idx: int | None
field: str
value: str
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
choice: Final = response.choices[0] if response.choices else None
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
blocks: Final = (
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
if fields is not None and isinstance(fields.get("web_search_results"), list)
else ()
)
return tuple(
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
)
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
fields: Final = as_str_mapping(result)
if fields is None or fields.get("type") != "web_search_result":
return ()
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
return tuple(
_WebSearchString(block_idx, result_idx, field, value)
for field, value in values
if isinstance(value, str) and value
)
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
block_id: Final = block.get("id")
match block.get("type"):
case "server_tool_use":
return True
case "tool_use":
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
case _:
return False
def _web_search_block_strings(
block_idx: int, block: Mapping[str, object], *, with_query: bool
) -> tuple[_WebSearchString, ...]:
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
``web_search_tool_result`` result's title and url."""
match block.get("type"):
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
tool_input: Final = as_str_mapping(block.get("input"))
query: Final = None if tool_input is None else tool_input.get("query")
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
case "web_search_tool_result":
return tuple(
chain.from_iterable(
_web_search_result_strings(block_idx, result_idx, result)
for result_idx, result in enumerate(_web_search_results(block))
)
)
case _:
return ()
def _rewritten_web_search_result(
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
) -> object:
fields: Final = as_str_mapping(result)
if fields is None:
return result
return {
**fields,
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
}
def _rewritten_web_search_block(
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
) -> Mapping[str, object]:
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
if _is_server_tool_use(block):
query: Final = by_location.get((None, "query"))
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
return block if query is None else {**block, "input": {**tool_input, "query": query}}
if not isinstance(block.get("content"), list):
return block
return {
**block,
"content": [
_rewritten_web_search_result(result_idx, result, by_location)
for result_idx, result in enumerate(_web_search_results(block))
],
}
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
block: Final = as_str_mapping(content_block)
if block is None or not isinstance(content_block, _WritableMessage):
return
rewritten: Final = _rewritten_web_search_block(block, rewrites)
writable: Final = _as_writable(content_block)
for key in ("input", "content"):
if key in rewritten:
writable[key] = rewritten[key]
def _split_web_search_texts(
guardrailed: Sequence[str], web_search_count: int, reply_count: int
) -> tuple[Sequence[str], Sequence[str]]:
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
return guardrailed[:split_at], guardrailed[split_at:]
def _web_search_rewrites_by_block(
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
rewrites: Final = tuple(
(target, value)
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
)
return MappingProxyType(
{
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
for block_idx in {target.block_idx for target, _ in rewrites}
}
)
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
return (
block.get("type") == "tool_use"
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
_DECODED_SSE_DATA: Final = TypeAdapter(object)
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
_RELEASED_TOOL_USE_STOP: Final = (
b"event: message_delta\n"
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
@ -315,6 +452,51 @@ def _write_back_tool_use(
block["name"] = shape.name
def _streamed_query_strings(
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
) -> tuple[_WebSearchString, ...]:
if len(tool_call_blocks) != len(tool_calls):
return ()
return tuple(
chain.from_iterable(
_web_search_block_strings(
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
)
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
)
)
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
return MappingProxyType(
{
target.block_idx: value
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
}
)
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
if arguments is None:
return tool_call
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
def _with_query_rewrites(
tool_calls: tuple[_ToolCallShape, ...],
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
query_rewrites: Mapping[int, str],
) -> tuple[_ToolCallShape, ...]:
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
return tool_calls
return tuple(
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
)
@dataclass(frozen=True, slots=True)
class _SSEFieldRewrite:
"""One field of one nested section of a buffered SSE event, rewritten."""
@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
def post_call_hook_response(self, response: object) -> object:
if not isinstance(response, ModelResponse):
return response
return self.adapter.translate_openai_response_to_anthropic(response)
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
result_blocks: Final = _streamed_web_search_result_blocks(response)
if not result_blocks:
return translated
content: Final = translated.get("content") or ()
return {**translated, "content": [*result_blocks, *content]}
@staticmethod
def _build_streaming_usage_response(
@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
task_mappings,
tool_calls_to_check,
)
block_fields: Final = tuple(map(self._output_block_fields, response_content))
web_search_strings: Final = tuple(
chain.from_iterable(
_web_search_block_strings(content_idx, fields[1], with_query=True)
for content_idx, fields in enumerate(block_fields)
if fields is not None
)
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_to_check or web_search_strings:
request_data = self._prepare_request_data(
request_data,
response,
@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
)
inputs: Final = self._build_guardrail_inputs(
texts_to_check,
[*(target.value for target in web_search_strings), *texts_to_check],
images_to_check,
tool_calls_to_check,
response,
@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
web_search_rewrites, reply_rewrites = _split_web_search_texts(
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
)
# Step 3: Map guardrail responses back to original response structure
await self._apply_guardrail_responses_to_output(
response=response,
responses=guardrailed_texts,
responses=list(reply_rewrites),
task_mappings=task_mappings,
)
self._write_back_web_search_strings(
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
)
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
guardrail_inputs: Final = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
if texts_to_check:
guardrail_inputs["texts"] = texts_to_check
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
responses_so_far, request_data
)
raise
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
web_search_rewrites, reply_rewrites = _split_web_search_texts(
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
)
if (
deliver_ended_stream_rewrites
and isinstance(string_so_far, str)
and string_so_far
and guardrailed_texts
and guardrailed_texts[0] != string_so_far
and reply_rewrites
and reply_rewrites[0] != string_so_far
):
self._write_ended_stream_text_rewrite(
responses_so_far,
guardrailed_texts[0],
reply_rewrites[0],
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
if deliver_ended_stream_rewrites:
self._write_ended_stream_web_search_rewrites(
responses_so_far,
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
)
if deliver_ended_stream_rewrites:
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
self._write_ended_stream_tool_call_rewrites(
responses_so_far,
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
post_guardrail_tool_calls=_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
post_guardrail_tool_calls=_with_query_rewrites(
_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
),
tool_call_blocks,
_query_rewrites(
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
),
),
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
request_data["litellm_metadata"] = user_metadata
return request_data
@staticmethod
def _write_back_web_search_strings(
response_content: Sequence[object],
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
for block_idx, rewrites in rewrites_by_block.items():
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
@staticmethod
def _get_response_content(response: object) -> Sequence[object]:
"""Extract content list from a dict or object response."""
@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
) -> None:
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
buffered chunks in place: the rebuilt response lists tool calls in the
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
on the nth block, its first ``input_json_delta`` carrying the full rewritten
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
arguments, every later one blanked, and ``content_block_start`` carrying the
rewritten name. Blocks that do not line up with the rebuilt tool calls make
the rewrite undeliverable, so the pipeline executor discards it and releases
the original chunks."""
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
block_indices: Final = tuple(
index
for item in responses_so_far
for event in cls._iter_sse_events(item)
if event.get("type") == "content_block_start"
and isinstance(block := event.get("content_block"), Mapping)
and block.get("type") == "tool_use"
and isinstance(index := event.get("index"), int)
)
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
if len(block_indices) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
f"{len(block_indices)} tool_use blocks",
f"{len(block_indices)} tool_use or server_tool_use blocks",
)
rewrites_by_block: Final = MappingProxyType(
{
@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
),
)
@classmethod
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
result_blocks: Final = (
(index, block)
for event in events
if event.get("type") == "content_block_start"
and isinstance(index := event.get("index"), int)
and (block := as_str_mapping(event.get("content_block"))) is not None
)
return tuple(
chain.from_iterable(
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
)
)
@classmethod
def _write_ended_stream_web_search_rewrites(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
if not rewrites_by_block:
return
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
index: Final = event.get("index")
block: Final = as_str_mapping(event.get("content_block"))
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
if event.get("type") != "content_block_start" or block is None or rewrites is None:
return None
return _SSEFieldRewrite(
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
@classmethod
def _streamed_tool_call_blocks(
cls, responses_so_far: Sequence[object]
) -> tuple[tuple[int, Mapping[str, object]], ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
return tuple(
(index, block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and isinstance(block_type := block.get("type"), str)
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
and isinstance(index := event.get("index"), int)
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
return StreamingScanKey(
texts=(self.get_streaming_string_so_far(responses_so_far),),
texts=(*(target.value for target in result_strings), streamed_text),
tool_calls=tool_use_fingerprints if stream_ended else (),
stream_ended=stream_ended,
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,

View file

@ -8,12 +8,13 @@ import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -24,6 +25,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
def _message_blocks(message: object) -> Sequence[object]:
fields: Final = as_str_mapping(message)
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
return blocks or ()
def _is_signed_compaction_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "compaction":
return False
signature: Final = fields.get("signature")
return isinstance(signature, str) and bool(signature)
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
if not isinstance(context_management, list):
return context_management
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
[entry for entry in context_management if isinstance(entry, dict)]
)
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
any(edit_type != "compact_20260112" for edit_type in edit_types),
),
)
if used
)
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drop_params,
)
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

View file

@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)

View file

@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(
self,
auth_header: Mapping[str, str],
messages: Sequence[Mapping[str, JsonValue]],
context_management: JsonValue = None,
) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
merges with the token-counting beta instead of replacing it."""
merges with the token-counting beta instead of replacing it. A history that carries a
signed compaction block also needs the compaction beta, or Anthropic rejects the block
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
compaction_betas: Final = (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
if carries_signed_compaction_block(messages)
else ()
)
request_betas: Final = (
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
*compaction_betas,
*context_management_betas(context_management),
)
return {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
),
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
}
def validate_request(

View file

@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
api_base: Final = extra_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
)

View file

@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
@ -255,7 +256,12 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
self,
kwargs,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -289,6 +295,7 @@ class AnthropicAdapter:
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
)
@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter:
)
@staticmethod
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
def _supports_prompt_cache_key(
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
) -> bool:
if not model or not custom_llm_provider:
return False
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter:
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "prompt_cache_key" in (supported_params or ())
if "prompt_cache_key" not in (supported_params or ()):
return False
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
return True
_, _, _, resolved_api_base = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
mistral_config: Final = litellm.MistralConfig()
return mistral_config.takes_prompt_cache_key_value(
prompt_cache_key
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
@staticmethod
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> None:
"""Translate metadata fields from Anthropic request to OpenAI request."""
if "metadata" in anthropic_message_request:
@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["user"] = metadata["user_id"]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None and self._supports_prompt_cache_key(
anthropic_message_request.get("model"), custom_llm_provider
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
):
new_kwargs["prompt_cache_key"] = prompt_cache_key
@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
## CONVERT TOOL CHOICE
self._translate_tool_choice_to_openai(

View file

@ -12,11 +12,20 @@ import json
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
citations: Final = block.get("citations")
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
class FakeAnthropicMessagesStreamIterator:
"""
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
chunks: Final = list[bytes]()
block_type: Final = block_dict.get("type")
if block_type == "text":
citations: Final = _citations_of(block_dict)
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
"content_block": {"type": "text", "text": "", "citations": ()}
if citations
else {"type": "text", "text": ""},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
citation_deltas: Final = (
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "citations_delta", "citation": citation},
}
for citation in citations
)
chunks.extend(
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
)
text: Final = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
elif block_type == "redacted_thinking":
content_block_start = {
elif block_type in ("tool_use", "server_tool_use"):
empty_input: Final[Mapping[str, object]] = {}
tool_use_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
"content_block": {**block_dict, "input": empty_input},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
input_data: Final = block_dict.get("input", {})
content_block_delta = {
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
tool_use_delta: Final = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
"partial_json": json.dumps(block_dict.get("input", {})),
},
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
else:
passthrough_start: Final = {

View file

@ -43,6 +43,7 @@ from ...common_utils import (
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
context_management_betas,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
if transformed_context_management is not None:
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
elif drops_unsupported_params(litellm_params.drop_params):
anthropic_messages_optional_request_params.pop("context_management")
####### get required params for all anthropic messages requests ######
# Lazy %s: the f-string previously stringified the entire messages
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

View file

@ -42,6 +42,7 @@ from ..openai.common_utils import (
build_output_token_limit_response,
is_output_token_limit_error,
model_dump_provider_response,
upstream_error_response,
)
from .common_utils import (
AzureOpenAIError,
@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
error_text = error_response.text
raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def make_async_azure_httpx_request(
self,
@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
error_code: Final[object] = getattr(e, "status_code", None)
if error_code is not None:
raise AzureOpenAIError(status_code=error_code, message=str(e))
raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e))
else:
raise AzureOpenAIError(status_code=500, message=str(e))

View file

@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
from ...openai.common_utils import upstream_error_response
from ...openai.completion.transformation import OpenAITextCompletionConfig
from ..common_utils import AzureOpenAIError, BaseAzureLLM
@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
async def acompletion(
self,
@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
def streaming(
self,
@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)

View file

@ -1,5 +1,7 @@
from typing import Final
import httpx
from litellm.exceptions import ContentPolicyViolationError
@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping:
model: str,
extra_information: str,
original_exception: Exception,
response: httpx.Response | None,
) -> ContentPolicyViolationError:
"""
Create a content policy violation error
@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping:
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
provider_specific_fields={
# Preserve legacy key for backward compatibility.
"innererror": inner_error,

View file

@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
def deployment_names_wrap_openai_models(self) -> bool:
return True
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).

View file

@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.prompt_templates.common_utils import (
content_parts,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
return any(_is_list_only_content_part(part) for part in content_parts(message))
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
content: Final = message.get("content")
if not isinstance(content, list):
return ()
try:
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
except ValidationError:
return ()
return tuple(part for part in parts if part.get("type") == "text")
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
return message
text_only: Final = {**message, "content": list(text_parts)}
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
return None
return PromptCacheBreakpoint(mode="explicit")
def _part_with_foundry_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _FIELDS.validate_python(part)
if "prompt_cache_breakpoint" not in fields:
return fields
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
parts: Final = content_parts(renamed) if drop_params else ()
if not parts:
return renamed
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
model_info: Final = litellm_params.get("model_info")
if not isinstance(model_info, Mapping):
return any(
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
)
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
if isinstance(deployment_flag, bool):
return deployment_flag
return any(
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
for name in named_models
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig):
self,
messages: list[AllMessageValues],
model: str,
reads_breakpoints: bool | None = None,
drop_params: bool = False,
) -> list:
"""
- Azure AI Studio doesn't support content as a list. This handles:
@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig):
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.
2. Transforms list content to a string.
2. Transforms list content to a string, except on deployments that read OpenAI explicit
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
content part can carry the breakpoint (a message-level one is silently ignored), so those
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
way the openai provider renames it. Every other part is dropped, exactly as the string
conversion drops it, before the marks are renamed, so a message-level mark lands on the
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
shape Foundry takes, and a malformed one is dropped.
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
deployment_reads_breakpoints: Final = (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
if reads_breakpoints is None
else reads_breakpoints
)
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
stripped_messages: Final = (
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
if deployment_reads_breakpoints
else copied_messages
)
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig):
if _has_list_only_content_part(message):
continue
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
message_dict["content"] = list(text_parts)
continue
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig):
if extra_body and isinstance(extra_body, dict):
optional_params.update(extra_body)
optional_params.pop("max_retries", None)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
return {
"model": model,
"messages": self._transform_messages(
messages,
model,
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
bool(litellm_params.get("drop_params") or litellm.drop_params),
),
**optional_params,
}
def transform_response(
self,

View file

@ -1,10 +1,12 @@
import dataclasses
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from types import TracebackType
from typing import TYPE_CHECKING, Any, Protocol
from typing import TYPE_CHECKING, Any, Literal, Protocol
import httpx
from litellm._uuid import uuid
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
@ -22,6 +24,26 @@ else:
LiteLLMLoggingObj = Any
@dataclasses.dataclass(frozen=True, slots=True)
class RealtimeSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
message: str
param: str
def error_event(self, client_event_id: object) -> dict[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": self.code,
"message": self.message,
"param": self.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
class RealtimeBackend(Protocol):
async def __aenter__(self) -> object: ...
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def builds_setup_from_session_update(self) -> bool:
return False
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None

View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
import httpx
@ -27,6 +27,20 @@ else:
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{"summary": REASONING_SUMMARY_VALUES}
)
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
)
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
)
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
)
class BaseResponsesAPIConfig(ABC):
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
def reasoning_field_values(self) -> ReasoningFieldValues | None:
return None
def deployment_names_wrap_openai_models(self) -> bool:
return False
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return None
def ignores_unsupported_reasoning_fields(self) -> bool:
return False
@abstractmethod
def map_openai_params(
self,

View file

@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import redact_string, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
from .transformation import BedrockRealtimeConfig, requested_modalities
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -256,23 +256,11 @@ async def _ack_session_update(
async def _reject_session_update(
client_ws: RealtimeClientWebSocket,
parsed_client_message: Mapping[str, JsonValue],
rejection: BedrockSessionUpdateRejection,
rejection: RealtimeSessionUpdateRejection,
) -> bool:
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
client_event_id: Final = parsed_client_message.get("event_id")
error_event: Final = {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": rejection.code,
"message": rejection.message,
"param": rejection.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
try:
await client_ws.send_text(json.dumps(error_event))
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
return False

View file

@ -11,7 +11,7 @@ import math
import uuid as uuid_lib
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal
from typing import Final
import httpx
from pydantic import ConfigDict, JsonValue, TypeAdapter
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.base import LiteLLMBaseModel
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockLockedSessionSettings:
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
max_tokens=max_tokens,
)
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
"""The error for a session.update this session cannot apply, or None when it applies."""
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if not self.prompt_started:
return None
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
param="session",

View file

@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
def custom_llm_provider(self) -> str:
return "bigmodel"
@property
def prompts_unsupported_json_schema(self) -> bool:
return False
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)

View file

@ -122,6 +122,12 @@ class Authenticator:
return tokens["access_token"]
def get_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=True)
def read_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=False)
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
auth_data: Final = self._read_auth_file()
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
return extract_chatgpt_account_id(access_token)
@ -133,7 +139,7 @@ class Authenticator:
id_token: Final = auth_data.get("id_token")
saved_access_token: Final = auth_data.get("access_token")
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
if derived:
if derived and persist_derived:
self._write_auth_file({**auth_data, "account_id": derived})
return derived

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
safe_convert_created_field,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""ChatGPT does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES

View file

@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
),
)
# Auto-send session setup if the provider requires it (e.g.
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
# Auto-send session setup at connect unless the provider builds it from the
# client's first session.update (Gemini/Vertex AI Live accept exactly one
# setup, so RealTimeStreaming sends theirs on the first client message).
# Build the streaming handler first so a transcription guardrail's
# auto-response disable can be folded into this one setup: Gemini
# rejects a second setup, so a follow-up disable would be dropped
# and the guardrail bypassed.
_session_config: str | None = None
if provider_config.requires_session_configuration():
sends_setup_at_connect: Final = (
provider_config.requires_session_configuration()
and not provider_config.builds_setup_from_session_update()
)
if sends_setup_at_connect:
_session_config = provider_config.session_configuration_request(model)
if _session_config:
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(

View file

@ -6,6 +6,10 @@ from urllib.parse import unquote
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.fireworks_ai.common_utils import (
FIREWORKS_FORWARD_USER_ID_PARAM,
get_fireworks_forwarded_user_id,
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_file_search(self) -> bool:
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

View file

@ -7,6 +7,7 @@ from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, Required, TypedDict
import litellm
@ -15,8 +16,9 @@ from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
{
"achernar",
"achird",
"algenib",
"algieba",
"alnilam",
"aoede",
"autonoe",
"callirrhoe",
"charon",
"despina",
"enceladus",
"erinome",
"fenrir",
"gacrux",
"iapetus",
"kore",
"laomedeia",
"leda",
"orus",
"puck",
"pulcherrima",
"rasalgethi",
"sadachbia",
"sadaltager",
"schedar",
"sulafat",
"umbriel",
"vindemiatrix",
"zephyr",
"zubenelgenubi",
}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
OpenAI stock voice names have no Gemini equivalent, so they are always
dropped, and so is any name outside Google's documented Live voices when
``drop_undocumented`` is set, keeping the model's default voice. Every
other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
if not isinstance(voice, str):
return VertexGeminiConfig().map_audio_params({"voice": voice})
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
if drop_undocumented and _is_undocumented_live_voice(voice):
verbose_logger.warning(
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig().map_audio_params({"voice": voice})
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
return max(len(data) * 3 // 4 - padding, 0)
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
def _chat_completion_tool(tool: object) -> dict[str, object]:
fields: Final = as_str_mapping(tool) or {}
if fields.get("type") != "function" or "function" in fields:
return {**fields}
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
parts: Final = system_instruction.get("parts", []) if system_instruction else []
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
return "\n".join(texts) if texts else None
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
def __init__(self):
def __init__(self, drop_params: bool = False):
super().__init__()
self.drop_params = drop_params
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
self._manual_activity_open: bool = False
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
return True
def _rejects_undocumented_voices(self, model: str) -> bool:
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
detail_mapping: Final = as_str_mapping(details)
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
except (json.JSONDecodeError, TypeError, AttributeError):
return False
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
return ()
self._manual_activity_open = True
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
return (json.dumps({"realtimeInput": activity_start}),)
def _end_manual_activity(self) -> tuple[str, ...]:
if not self._manual_activity_open:
return ()
self._manual_activity_open = False
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
return (json.dumps({"realtimeInput": activity_end}),)
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
if self._manual_turn_detection_enabled(session_configuration_request):
realtime_input_dict: BidiGenerateContentRealtimeInput = {
"activityEnd": True,
}
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
else:
realtime_input_dict = {"audioStreamEnd": True}
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
return [json.dumps({"realtimeInput": realtime_input_dict})]
return list(self._end_manual_activity())
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
return [json.dumps({"realtimeInput": audio_stream_end})]
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"voice",
]
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
if "generationConfig" not in optional_params:
optional_params["generationConfig"] = {}
for key, value in non_default_params.items():
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
elif key == "temperature":
optional_params["generationConfig"]["temperature"] = value
elif key == "max_response_output_tokens":
elif key == "max_response_output_tokens" and isinstance(value, int):
optional_params["generationConfig"]["maxOutputTokens"] = value
elif key == "modalities":
optional_params["generationConfig"]["responseModalities"] = [
modality.upper() for modality in cast(list[str], value)
]
optional_params["generationConfig"]["responseModalities"] = list(
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
)
elif key == "tools":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
vertex_gemini_config = VertexGeminiConfig()
# Tools should be at the top level of setup, not inside generationConfig
optional_params["tools"] = vertex_gemini_config.map_function(
value=value, optional_params=optional_params
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
optional_params=optional_params,
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
continue
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
if transformed_audio_activity_config:
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "turn_detection":
realtime_input_config = self._realtime_input_config(cast(object, value))
if realtime_input_config is not None:
optional_params["realtimeInputConfig"] = realtime_input_config
elif key == "voice":
speech_config = _gemini_live_speech_config(value)
speech_config = _gemini_live_speech_config(
value, self.drop_params and self._rejects_undocumented_voices(model)
)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
optional_params.pop("generationConfig")
return optional_params
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
if turn_detection is None:
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
return None
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
if not transformed_audio_activity_config:
return None
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
@staticmethod
def _extract_turn_detection(session: dict) -> dict | None:
"""Extract turn_detection from a session.update payload.
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(input_cfg, dict):
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
normalized["input_audio_transcription"] = input_cfg["transcription"]
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
normalized["turn_detection"] = input_cfg["turn_detection"]
output_cfg: Final = audio.get("output")
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
normalized["voice"] = output_cfg["voice"]
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return normalized
@staticmethod
def _model_cost_entry(model: str) -> dict:
def _model_cost_entry(model: str) -> Mapping[str, object]:
entry = litellm.model_cost.get(model)
if entry is None:
stripped: Final = model.split("/", 1)[-1]
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
def _requested_response_modalities(value: object) -> tuple[str, ...]:
if isinstance(value, str):
return (value.upper(),)
if not isinstance(value, (list, tuple)):
return ()
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
@staticmethod
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
if unknown:
return unknown
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
return ("TEXT",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
return ("AUDIO",)
return ()
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
if self.drop_params:
return None
if setup_sent:
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message=(
"Gemini Live accepts one session configuration per connection, and this session already sent "
"its configuration, so this session.update was not applied. Put every setting in session.update "
"before the first conversation content, or set drop_params to drop it instead"
),
param="session",
)
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
@staticmethod
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
)
if not unsupported:
return None
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.{param}': {model} cannot produce "
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
),
param=f"session.{param}",
)
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
output_voice: Final = audio_output.get("voice")
voice: Final = output_voice or session.get("voice")
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
return None
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
"Set drop_params to drop it instead"
),
param=param,
)
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
text-only ones such as transcribe-live), then keep a single modality
because Gemini Live rejects a setup naming two."""
return GeminiRealtimeConfig._single_response_modality(
GeminiRealtimeConfig._swap_unsupported_response_modalities(
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
)
)
@staticmethod
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
normalized: Final = tuple(str(modality).upper() for modality in modalities)
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
if len(known) < len(normalized):
verbose_logger.warning(
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
model,
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
)
return known or (GeminiRealtimeConfig._default_response_modality(model),)
@staticmethod
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
verbose_logger.warning(
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
model,
)
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
if len(modalities) <= 1:
return modalities
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
verbose_logger.warning(
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
chosen,
", ".join(modalities),
)
return (chosen,)
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
generation_config: Final = setup.get("generationConfig")
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_payload = self._normalize_session_payload_for_mapping(session_payload)
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
new_overrides: Final = self.map_openai_params(
optional_params={}, non_default_params=session_payload, model=model
)
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
message_str = str(message)
raise ValueError(f"Invalid JSON message: {message_str}")
messages: Final[list[str]] = []
msg_type: Final = json_message.get("type")
if msg_type == "session.update":
return self._handle_session_update(json_message, model, session_configuration_request)
if msg_type == "response.create":
return [] # Gemini responds automatically; nothing to forward
return list(self._end_manual_activity())
if msg_type == "conversation.item.create":
return self._handle_conversation_item(json_message)
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
messages.append(gemini_msg)
return messages
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
logging_session_id: str,
session_configuration_request: str | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
session_configuration_request_dict: Final = _parse_setup(
session_configuration_request or self.session_configuration_request(model)
)
_model: Final = session_configuration_request_dict.get("model") or model
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
session: Final = OpenAIRealtimeStreamSession(
id=logging_session_id,
modalities=_modalities,
)
if _system_instruction is not None and isinstance(_system_instruction, str):
session["instructions"] = _system_instruction
if instructions is not None:
session["instructions"] = instructions
if _model is not None and isinstance(_model, str):
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
if "/models/" in _model:
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
return not litellm.gemini_live_defer_setup
def builds_setup_from_session_update(self) -> bool:
return True
def session_configuration_request(self, model: str) -> str:
"""

View file

@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True
def supports_native_websocket(self) -> bool:
"""GitHub Copilot does not support native WebSocket for Responses API"""
return False

View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.LITELLM_PROXY
def reasoning_field_values(self) -> ReasoningFieldValues:
return ANY_REASONING_FIELD_VALUES
def get_complete_url(
self,
api_base: str | None,

View file

@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from itertools import accumulate, count
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import (
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.mistral import (
MistralCacheKeyParams,
MistralFilePart,
MistralReasoningEffortParams,
MistralRequestRouting,
MistralTextBlock,
MistralThinkingBlock,
MistralToolCallMessage,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
requested: Final = as_str_mapping(reasoning_effort)
if requested is None:
return reasoning_effort, None
return requested.get("effort"), requested.get("summary")
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig):
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
"""
_is_base_class = False
temperature: int | None = None
top_p: int | None = None
max_tokens: int | None = None
@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
is_magistral: Final = "magistral" in model.lower()
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
return [
"stream",
"temperature",
@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig):
"parallel_tool_calls",
*(("thinking",) if is_magistral else ()),
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
]
def _map_tool_choice(self, tool_choice: str) -> str:
@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig):
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
optional_params["response_format"] = value
if param == "reasoning_effort" and "magistral" not in model.lower():
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
if (
param == "reasoning_effort"
and "magistral" not in model.lower()
and (
reasoning_level := self._reasoning_level_to_send(
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
)
)
is not None
):
optional_params["reasoning_effort"] = reasoning_level
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
if param == "parallel_tool_calls":
optional_params["parallel_tool_calls"] = value
if param in ("parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
return sendable
raise UnsupportedParamsError(
message=(
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=self.custom_llm_provider,
model=model,
)
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
hostname: Final = urlparse(resolved_api_base).hostname or ""
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
@staticmethod
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
if not self.accepts_prompt_cache_key_at(api_base):
return "since only the Mistral API (*.mistral.ai) takes it"
if not self.takes_prompt_cache_key_value(prompt_cache_key):
return "since the Mistral API takes it only as a string of ASCII characters"
return None
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
requested_api_base: Final = routing.get("api_base")
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
allowed_openai_params: Final = routing.get("allowed_openai_params")
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
return False
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
if rejection is None:
return False
if litellm.drop_params is True or routing.get("drop_params") is True:
return True
requested_provider: Final = routing.get("custom_llm_provider")
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
raise UnsupportedParamsError(
message=(
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=provider,
model=model,
)
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
api_base = (
@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig):
messages = self._handle_message_with_file(messages)
return messages
@staticmethod
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
file_part: Final = block.get("file") if block.get("type") == "file" else None
file_id: Final = (
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
)
if not file_id:
return block
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
@classmethod
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
content: Final = message.get("content")
if not isinstance(content, list):
return message
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
)
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
"""
Mistral API supports only 'file_id' in message content with type 'file'.
"""
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
if any(c.get("type") == "file" for c in _content_block):
# If file content is present, we get file_id from 'file' attribute of content block
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
file_contents = [c for c in _content_block if c.get("type") == "file"]
for file_content in file_contents:
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
file_content.pop("file", None)
return messages
return [self._with_mistral_file_ids(m) for m in messages]
def _add_reasoning_system_prompt_if_needed(
self, messages: list[AllMessageValues], optional_params: dict
@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig):
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
withheld_params: Final = (
("client_metadata", "prompt_cache_key")
if "prompt_cache_key" in cache_key_params
and self._prompt_cache_key_withheld(
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
)
else ("client_metadata",)
)
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
# Call parent transform_request which handles _transform_messages
return super().transform_request(

View file

@ -29,6 +29,7 @@ if TYPE_CHECKING:
from aiohttp import ClientSession
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
@ -148,6 +149,14 @@ class OpenAIError(BaseLLMException):
)
def upstream_error_response(error: Exception) -> httpx.Response | None:
if not isinstance(error, (openai.APIStatusError, httpx.HTTPStatusError)):
return None
return error_response_without_credentials(
error.response.status_code, error.response, "https://docs.litellm.ai/docs"
)
_DECODED_JSON: Final = TypeAdapter(object)

View file

@ -10,7 +10,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
from litellm.utils import ProviderConfigManager
from ..common_utils import BaseOpenAILLM, OpenAIError
from ..common_utils import BaseOpenAILLM, OpenAIError, upstream_error_response
from ..workload_identity import build_async_openai_client, build_openai_client
from .transformation import OpenAITextCompletionConfig
@ -164,7 +164,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def acompletion(
self,
@ -220,7 +225,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def streaming(
self,
@ -261,7 +271,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: object = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=response,
model=model,

View file

@ -14,7 +14,7 @@ from litellm.utils import ProviderConfigManager
from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, upstream_error_response
from ..workload_identity import build_async_openai_client, build_openai_client
@ -129,7 +129,12 @@ class OpenAIImageVariationsHandler:
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def image_variations(
self,
@ -245,4 +250,9 @@ class OpenAIImageVariationsHandler:
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)

View file

@ -27,6 +27,7 @@ import litellm
from litellm import LlmProviders
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.exceptions import UnsupportedParamsError
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -64,6 +65,7 @@ from .common_utils import (
is_openai_backed_api_base,
is_output_token_limit_error,
model_dump_provider_response,
upstream_error_response,
)
from .workload_identity import (
build_async_openai_client,
@ -970,8 +972,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
litellm.remove_index_from_tool_calls(messages=messages)
else:
raise e
except OpenAIError as e:
raise e
except (OpenAIError, UnsupportedParamsError):
raise
except Exception as e:
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
@ -983,6 +985,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -1110,6 +1113,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
@ -1282,6 +1286,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=getattr(e, "status_code", 500),
message=str(e),
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
@ -1409,7 +1414,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def embedding(
self,
@ -1499,7 +1509,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def aimage_generation(
self,
@ -1666,7 +1681,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
original_response=str(e),
)
if hasattr(e, "status_code"):
raise OpenAIError(status_code=getattr(e, "status_code", 500), message=str(e))
raise OpenAIError(
status_code=getattr(e, "status_code", 500),
message=str(e),
response=upstream_error_response(e),
)
else:
raise OpenAIError(status_code=500, message=str(e))

View file

@ -9,6 +9,7 @@ The only provider quirks:
Ref: https://docs.perplexity.ai/api-reference/responses-post
"""
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
_DECODED_JSON: Final = TypeAdapter(object)
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
)
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""Perplexity does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return PERPLEXITY_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

View file

@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import AWSEventStreamDecoder, SagemakerError
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
inference_component_name: Final = optional_params.get("model_id")
if not isinstance(inference_component_name, str):
return headers
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
other_headers: Final = {
name: value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
}
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
def transform_request(
self,
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
litellm_params=litellm_params,
headers=headers,
)
container_request: Final = {
key: value for key, value in request.items() if key not in self.aws_authentication_params
}
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
served_model_name: Final = litellm_params.get("hf_model_name")
if not isinstance(served_model_name, str):
return container_request

View file

@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
from litellm import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
return _load_sagemaker_response_stream_shape()
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
_SAGEMAKER_PROVIDERS: Final = frozenset(
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
)
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if isinstance(custom_llm_provider, str) and custom_llm_provider:
return custom_llm_provider
model: Final = litellm_params.get("model")
return model.split("/", 1)[0] if isinstance(model, str) else None
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
headers: Final = litellm_params.get("extra_headers")
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
return None
return next(
(
value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
),
None,
)
def with_inference_component_hint(error_message: str) -> str:
if "Inference Component Name header is required" in error_message:
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

View file

@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import token_counter
from ..common_utils import SagemakerError
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
inference_params: Final = optional_params.copy()
inference_params: Final = {
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
}
stream: Final = inference_params.pop("stream", False)
data: Final[dict] = {"parameters": inference_params}
if stream is True:

Some files were not shown because too many files have changed in this diff Show more