mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_drop_malformed_values
# Conflicts: # litellm/responses/utils.py
This commit is contained in:
commit
73d5ced9c5
258 changed files with 16864 additions and 4496 deletions
152
.github/workflows/lens-install-smoke.yml
vendored
152
.github/workflows/lens-install-smoke.yml
vendored
|
|
@ -1,152 +0,0 @@
|
|||
name: Lens installation smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
lens_image:
|
||||
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
lens_upgrade_image:
|
||||
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
gateway_baseline_ref:
|
||||
description: Previously qualified gateway source commit, full SHA
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lens-install-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
- component: ui
|
||||
dockerfile: ui/Dockerfile
|
||||
- component: migrations
|
||||
dockerfile: migrations/Dockerfile
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
baseline: true
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
baseline: true
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
baseline: true
|
||||
env:
|
||||
COMPONENT: ${{ matrix.component }}
|
||||
DOCKERFILE: ${{ matrix.dockerfile }}
|
||||
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
|
||||
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
|
||||
CURRENT_REF: ${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.SOURCE_REF }}
|
||||
- name: Verify the selected source
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
|
||||
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
|
||||
test "$SOURCE_REF" != "$CURRENT_REF"
|
||||
fi
|
||||
- name: Build the matching release image
|
||||
run: |
|
||||
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
|
||||
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
|
||||
- name: Save the matching release image
|
||||
run: |
|
||||
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
|
||||
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
|
||||
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
|
||||
compression-level: 0
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
|
||||
helm-install:
|
||||
needs: build-images
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: lens-install-*-${{ github.sha }}
|
||||
merge-multiple: true
|
||||
path: ${{ runner.temp }}/lens-install-images
|
||||
- name: Load the matching release images
|
||||
run: |
|
||||
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
|
||||
gzip -dc "$archive" | docker load
|
||||
rm "$archive"
|
||||
done
|
||||
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
|
||||
with:
|
||||
cosign-release: v2.4.3
|
||||
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ vars.LENS_READ_USER || github.actor }}
|
||||
password: ${{ secrets.LENS_READ_TOKEN }}
|
||||
- name: Verify independent Lens images before loading them
|
||||
env:
|
||||
LENS_IMAGE: ${{ inputs.lens_image }}
|
||||
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
|
||||
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
|
||||
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
|
||||
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
|
||||
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
|
||||
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
|
||||
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
|
||||
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
|
||||
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
|
||||
docker pull "$image"
|
||||
done
|
||||
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
|
||||
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
|
||||
- name: Install pinned Kubernetes test tools
|
||||
run: |
|
||||
curl --fail --location --output "$RUNNER_TEMP/kind" \
|
||||
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
|
||||
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kind"
|
||||
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
|
||||
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
|
||||
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kubectl"
|
||||
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
|
||||
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
|
||||
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
|
||||
- name: Install, ingest, upgrade, and restart both charts
|
||||
run: bash tests/e2e/migrations/lens_helm_smoke.sh
|
||||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -227,6 +227,7 @@ jobs:
|
|||
tests/unit/llms/test_oss_decision.py
|
||||
tests/unit/llms/test_polling_url_origin_match.py
|
||||
tests/unit/llms/test_predibase_transformation.py
|
||||
tests/unit/llms/test_subscription_accounts.py
|
||||
tests/unit/llms/a2a
|
||||
tests/unit/llms/aiml
|
||||
tests/unit/llms/aiohttp_openai
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/cache_settings",
|
||||
"/coordination_redis/",
|
||||
"/cost_tracking",
|
||||
"/subscription_accounts/",
|
||||
"/cost_optimization/",
|
||||
"/cost/",
|
||||
"/credentials",
|
||||
|
|
|
|||
|
|
@ -5,8 +5,5 @@ dependencies:
|
|||
- name: redis
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
version: 18.19.1
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
|
||||
generated: '2026-10-09T04:18:10.643228000Z'
|
||||
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
|
||||
generated: '2026-10-30T02:13:11.505726000Z'
|
||||
|
|
|
|||
|
|
@ -39,6 +39,3 @@ dependencies:
|
|||
version: "18.19.1"
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
condition: redis.enabled
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -256,9 +256,7 @@ itself.
|
|||
|
||||
## Connect Lens
|
||||
|
||||
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
## Shutdown accounting
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
|
|||
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -402,95 +398,28 @@ shutdown drain window.
|
|||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.labels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
|
||||
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
{{- if .Values.ingress.enabled -}}
|
||||
{{- $fullName := include "litellm.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
{{- $lensMode := include "litellm.lens.mode" . -}}
|
||||
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
|
||||
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
|
||||
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
|
||||
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
|
||||
|
|
@ -46,18 +44,18 @@ spec:
|
|||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
|
||||
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
name: {{ $.Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ $.Values.lensWorker.service.port }}
|
||||
number: {{ $.Values.lens.service.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
servicePort: {{ $.Values.lensWorker.service.port }}
|
||||
serviceName: {{ $.Values.lens.service.name }}
|
||||
servicePort: {{ $.Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .paths }}
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
- ingress.yaml
|
||||
tests:
|
||||
- it: connects the proxy to an external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service ahead of the host paths
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[0].path
|
||||
value: /lens-ingest
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- ingress.yaml
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects deployment.yaml to the shared Lens service
|
||||
template: deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.hosts:
|
||||
- host: gateway.example
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in deployment.yaml
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
|
@ -668,70 +668,15 @@ serviceMonitor:
|
|||
# - test-namespace
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
dependencies:
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
|
||||
generated: '2026-10-09T04:18:10.642706000Z'
|
||||
|
|
@ -6,8 +6,3 @@ version: 0.1.0
|
|||
appVersion: "0.1.0"
|
||||
annotations:
|
||||
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
|
||||
|
||||
dependencies:
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
# LiteLLM componentized chart
|
||||
|
||||
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
|
||||
This chart deploys separate gateway, backend and UI services
|
||||
|
||||
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
|
|||
collector containers through an emptyDir. Empty when the sidecar is off
|
||||
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
|
||||
*/}}
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -635,93 +631,27 @@ shutdown drain window.
|
|||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -166,14 +166,14 @@ spec:
|
|||
- host: {{ $componentHosts.gateway | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
|
|
@ -249,15 +249,15 @@ spec:
|
|||
port:
|
||||
number: {{ $gatewayPort }}
|
||||
{{- end }}
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- /*
|
||||
--- Operator-supplied extra paths (ingress.extraPaths) ---
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
|
|
@ -15,7 +15,6 @@ tests:
|
|||
ingress.componentHosts.ui: ui.example.com
|
||||
ingress.componentHosts.backend: backend.example.com
|
||||
ingress.componentHosts.gateway: gateway.example.com
|
||||
lensWorker.mode: disabled
|
||||
ingress.tls:
|
||||
- hosts:
|
||||
- ui.example.com
|
||||
|
|
@ -83,12 +82,12 @@ tests:
|
|||
ui.backendUrl: https://backend.example.com
|
||||
ui.publicUrl: https://ui.example.com
|
||||
backend.gatewayUrl: https://gateway.example.com
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.externalUrl: https://lens.example.com
|
||||
lensWorker.publicUrl: https://lens.example.com
|
||||
lensWorker.serviceTokenSecret.name: existing-lens-signing
|
||||
lensWorker.gateway.secretName: existing-lens-signing
|
||||
lens.enabled: true
|
||||
lens.service.name: existing-lens
|
||||
lens.url: http://existing-lens:4318
|
||||
lens.publicUrl: https://gateway.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: existing-lens-signing
|
||||
lens.gatewaySecret.name: existing-lens-signing
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[2].http.paths[0].path
|
||||
|
|
|
|||
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: connects the gateway to an external Lens
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: connects the backend to an external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service on the single ingress host
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: keeps the ingestion endpoint on an independently deployed Lens service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects gateway/deployment.yaml to the shared Lens service
|
||||
template: gateway/deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: connects backend/deployment.yaml to the shared Lens service
|
||||
template: backend/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: omits Lens connection settings when disabled in backend/deployment.yaml
|
||||
template: backend/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: gateway/deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,164 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
suite: Independent Lens release and credentials
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
tests:
|
||||
- it: selects a Lens release independently of the gateway
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
lensWorker.image.tag: 2.3.4
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:2.3.4
|
||||
- it: retains the pinned Lens chart default when only the gateway changes
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:0.1.0-dev.0
|
||||
- it: selects development publications explicitly
|
||||
set:
|
||||
lensWorker.image.repository: ghcr.io/berriai/lens-dev
|
||||
lensWorker.image.tag: sha-0123456789abcdef
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
|
||||
- it: preserves a verified registry mirror and digest
|
||||
set:
|
||||
lensWorker.image.repository: registry.example/lens
|
||||
lensWorker.image.tag: replaced-label
|
||||
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
- it: rejects a malformed digest
|
||||
set:
|
||||
lensWorker.image.digest: sha256:invalid
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
|
||||
- it: keeps the legacy enablement switch working
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments: {count: 0}
|
||||
- it: keeps isolated local analysis with delegated gateway access
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: LENS_MODE, value: standalone}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].emptyDir
|
||||
value: {medium: Memory, sizeLimit: 1Gi}
|
||||
|
|
@ -654,70 +654,15 @@ ui:
|
|||
topologySpreadConstraints: []
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
|
||||
"subscription_account_id" TEXT NOT NULL,
|
||||
"custom_llm_provider" TEXT NOT NULL,
|
||||
"account_id" TEXT NOT NULL,
|
||||
"label" TEXT,
|
||||
"monthly_fee" DOUBLE PRECISION NOT NULL,
|
||||
"currency" TEXT NOT NULL,
|
||||
"billing_period_start" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT NOT NULL,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_by" TEXT NOT NULL,
|
||||
|
||||
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");
|
||||
|
|
@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
|
|||
@@id([event_id, target, version])
|
||||
@@index([target, state, next_attempt_at])
|
||||
}
|
||||
|
||||
model LiteLLM_SubscriptionAccountTable {
|
||||
subscription_account_id String @id @default(uuid())
|
||||
custom_llm_provider String
|
||||
account_id String
|
||||
label String?
|
||||
monthly_fee Float
|
||||
currency String
|
||||
billing_period_start String
|
||||
created_at DateTime @default(now()) @map("created_at")
|
||||
created_by String
|
||||
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
|
||||
updated_by String
|
||||
|
||||
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "together_ai":
|
||||
if _is_past_deprecation_date(value):
|
||||
together_ai_models.discard(key)
|
||||
elif not key.startswith("together-ai-"):
|
||||
together_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "baseten":
|
||||
if _is_past_deprecation_date(value):
|
||||
baseten_models.discard(key)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import Final, Literal
|
|||
|
||||
from litellm.litellm_core_utils.env_utils import (
|
||||
get_env_float,
|
||||
get_env_float_in_range,
|
||||
get_env_float_or_none,
|
||||
get_env_int,
|
||||
get_env_int_in_range,
|
||||
|
|
@ -67,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
|
|||
"retry_after",
|
||||
"fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"retry_policy",
|
||||
"model_group_retry_policy",
|
||||
"model_group_alias",
|
||||
|
|
@ -149,6 +151,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
|
|||
"DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
|
||||
) # default cooldown a deployment if 50% of requests fail in a given minute
|
||||
DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
|
||||
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
|
||||
"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
|
||||
)
|
||||
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
|
||||
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
|
||||
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
|
||||
|
|
@ -416,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
|
|||
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
|
||||
)
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
|
||||
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
|
||||
|
||||
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
|
@ -893,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
|
||||
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
|
||||
|
|
@ -1270,7 +1278,7 @@ empower_models: Final = set(
|
|||
]
|
||||
)
|
||||
|
||||
together_ai_models: Final[set] = set(
|
||||
together_ai_models: Final[set[str]] = set(
|
||||
[
|
||||
# llama llms - chat
|
||||
"togethercomputer/llama-2-70b-chat",
|
||||
|
|
|
|||
|
|
@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost(
|
|||
return litellm_logging_obj.get_response_ms()
|
||||
|
||||
|
||||
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
|
||||
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
|
||||
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
|
||||
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost(
|
|||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=None,
|
||||
completion_tokens_cost_usd_dollar=None,
|
||||
cost_for_built_in_tools_cost_usd_dollar=None,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=adjusted_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
discount_percent=discount_percent,
|
||||
|
|
@ -2376,6 +2382,15 @@ def response_cost_calculator(
|
|||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=provider_response_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=provider_response_cost,
|
||||
original_cost=provider_response_cost,
|
||||
)
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
## LiteLLM versions of the OpenAI Exception Types
|
||||
|
||||
import datetime
|
||||
import enum
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
|
@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _upstream_body(response: object) -> bytes:
|
||||
if not isinstance(response, httpx.Response):
|
||||
return b""
|
||||
try:
|
||||
return response.content
|
||||
except httpx.ResponseNotRead:
|
||||
return b""
|
||||
|
||||
|
||||
def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
|
||||
if not isinstance(upstream, httpx.Response):
|
||||
return httpx.Request(method="POST", url=fallback_url)
|
||||
try:
|
||||
upstream_request: Final = upstream.request
|
||||
except RuntimeError:
|
||||
return httpx.Request(method="POST", url=fallback_url)
|
||||
return httpx.Request(
|
||||
method=upstream_request.method,
|
||||
url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
|
||||
)
|
||||
|
||||
|
||||
def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
|
||||
if not isinstance(upstream, httpx.Response):
|
||||
return None
|
||||
try:
|
||||
return upstream.elapsed
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
|
||||
def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
|
||||
response: Final = httpx.Response(
|
||||
status_code=status_code,
|
||||
stream=httpx.ByteStream(_upstream_body(upstream)),
|
||||
request=_request_without_credentials(upstream, fallback_url),
|
||||
)
|
||||
_ = response.read()
|
||||
response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
|
||||
elapsed: Final = _upstream_elapsed(upstream)
|
||||
if elapsed is not None:
|
||||
response.elapsed = elapsed
|
||||
return response
|
||||
|
||||
|
||||
_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
|
||||
|
||||
|
||||
_MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
|
||||
|
||||
|
||||
|
|
@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
|
|||
# headers stay reachable on `e.response.headers` for callers that
|
||||
# explicitly want them; only the proxy-supplied `headers=` kwarg
|
||||
# makes it onto `self.headers`.
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
|
||||
# Mirrors FastAPI HTTPException.detail so the same instance can be
|
||||
# serialized through both the ProxyException and HTTPException paths.
|
||||
self.detail = detail if detail is not None else self.message
|
||||
self.response = httpx.Response(
|
||||
status_code=429,
|
||||
headers=_response_headers,
|
||||
content=response.content if response is not None else None,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message, response=self.response, body=None
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
|
@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -908,6 +924,7 @@ class APIError(openai.APIError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: object | None = None,
|
||||
response: httpx.Response | None = None,
|
||||
):
|
||||
self.status_code = status_code
|
||||
self.message = f"litellm.APIError: {message}"
|
||||
|
|
@ -916,6 +933,8 @@ class APIError(openai.APIError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
if response is not None:
|
||||
self.response = response
|
||||
if request is None:
|
||||
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
|
||||
super().__init__(self.message, request=request, body=body)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.integrations.prompt_management_base import PromptManagementClient
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
|
|||
|
||||
|
||||
def _hosted_openai_dialect_flag(
|
||||
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
resolve_provider: Callable[[str], str | None],
|
||||
model_cost: Mapping[str, Mapping[str, object]] | None = None,
|
||||
) -> bool | None:
|
||||
"""
|
||||
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
|
||||
|
|
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
|
|||
"""
|
||||
import litellm
|
||||
|
||||
exact_entry: Final = litellm.model_cost.get(model)
|
||||
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
|
||||
cost_map: Final = (
|
||||
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
|
||||
if model_cost is None
|
||||
else model_cost
|
||||
)
|
||||
exact_entry: Final = cost_map.get(model)
|
||||
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
|
||||
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
|
||||
return None
|
||||
provider: Final = custom_llm_provider or resolve_provider(model)
|
||||
if provider is None or provider == "openai":
|
||||
return None
|
||||
if isinstance(exact_entry, dict) and exact_provider == provider:
|
||||
if isinstance(exact_entry, Mapping) and exact_provider == provider:
|
||||
return _hosted_entry_flag(exact_entry, lambda: provider)
|
||||
from litellm.utils import get_potential_model_names
|
||||
|
||||
|
|
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
|
|||
names["stripped_model_name"],
|
||||
names["provider_prefixed_model_name"],
|
||||
)
|
||||
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
|
||||
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
|
||||
entries: Final = (cost_map.get(candidate) for candidate in candidates)
|
||||
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
|
||||
return next((flag for flag in flags if flag is not None), None)
|
||||
|
||||
|
||||
|
|
@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return False
|
||||
return prompt_cache_options is not None or targets_openai_api(api_base)
|
||||
|
||||
@staticmethod
|
||||
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
|
||||
return (
|
||||
_hosted_openai_dialect_flag(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
AnthropicCacheControlHook._resolve_provider,
|
||||
GetModelCostMap.loaded_model_cost_map(),
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
return _served_provider_for_model(model)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
if not isinstance(metadata, dict):
|
||||
return []
|
||||
return metadata.get("guardrails") or []
|
||||
|
||||
|
||||
class CustomGuardrail(CustomLogger):
|
||||
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
|
||||
use_native_during_call_hook: ClassVar[bool] = False
|
||||
|
|
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
|
|||
return True
|
||||
raise
|
||||
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
"""
|
||||
Returns the guardrail(s) to be run from the metadata or root
|
||||
Returns the guardrail(s) to be run from the root, else from both metadata fields together
|
||||
"""
|
||||
|
||||
if "guardrails" in data:
|
||||
return data["guardrails"]
|
||||
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
|
||||
# writes to "metadata"; for thread/assistant endpoints it writes to
|
||||
# "litellm_metadata". We check the one that actually contains the "guardrails"
|
||||
# key so that a non-empty litellm_metadata without guardrails does not shadow
|
||||
# the merged list stored in metadata (which would cause team guardrails to be
|
||||
# silently skipped while default_on=True policy guardrails still fire).
|
||||
for meta_key in ("metadata", "litellm_metadata"):
|
||||
meta = data.get(meta_key) or {}
|
||||
if isinstance(meta, dict) and "guardrails" in meta:
|
||||
return meta.get("guardrails") or []
|
||||
return []
|
||||
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
|
||||
return [
|
||||
*_guardrails_listed_in(request_fields.get("metadata")),
|
||||
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
|
||||
]
|
||||
|
||||
def _guardrail_is_in_requested_guardrails(
|
||||
self,
|
||||
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
|
||||
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
|
||||
) -> bool:
|
||||
for _guardrail in requested_guardrails:
|
||||
if isinstance(_guardrail, dict):
|
||||
|
|
|
|||
|
|
@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_LensReview",
|
||||
"LiteLLM_LensIngestionKey",
|
||||
"LiteLLM_LensWorker",
|
||||
"LiteLLM_SubscriptionAccountTable",
|
||||
"LiteLLM_LensSignalConfig",
|
||||
"LiteLLM_LensTraceSignal",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
(t for t in tools if is_anthropic_native_web_search_tool(t)),
|
||||
None,
|
||||
)
|
||||
emit_native_blocks: Final = native_tool is not None or (
|
||||
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
|
||||
)
|
||||
|
||||
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
|
||||
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
|
||||
|
||||
content: Final[list[dict[str, object]]] = []
|
||||
if native_tool is not None:
|
||||
if emit_native_blocks:
|
||||
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
|
||||
tool_name: Final = native_tool.get("name") or "web_search"
|
||||
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
|
||||
content.append(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
|
|
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
}
|
||||
)
|
||||
content.append(
|
||||
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
|
||||
WebSearchTransformation.build_web_search_outcome_block(
|
||||
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
|
||||
)
|
||||
)
|
||||
# Keep the text block so non-native short-circuit callers (Claude Code,
|
||||
# github_copilot, etc.) see the same payload they always have.
|
||||
content.append({"type": "text", "text": search_result_text})
|
||||
|
||||
response: Final[dict[str, object]] = {
|
||||
|
|
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
|
||||
len(search_result_text),
|
||||
native_tool is not None,
|
||||
emit_native_blocks,
|
||||
)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -412,6 +412,7 @@ class WebSearchTransformation:
|
|||
def build_web_search_tool_result_block(
|
||||
tool_use_id: str,
|
||||
search_response: SearchResponse | None,
|
||||
with_snippets: bool = True,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Build an Anthropic-native ``web_search_tool_result`` content block.
|
||||
|
|
@ -445,6 +446,8 @@ class WebSearchTransformation:
|
|||
``litellm.asearch()``. If None or empty, the block is still
|
||||
emitted with an empty result list (signals "search ran, no
|
||||
results" rather than "search did not run").
|
||||
with_snippets: False when a text block in the same reply already
|
||||
carries the snippets.
|
||||
"""
|
||||
items: Final[list[dict[str, object]]] = []
|
||||
if search_response is not None:
|
||||
|
|
@ -460,7 +463,7 @@ class WebSearchTransformation:
|
|||
"title": title,
|
||||
"page_age": page_age,
|
||||
"encrypted_content": "",
|
||||
"snippet": getattr(r, "snippet", "") or "",
|
||||
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
|
||||
}
|
||||
)
|
||||
return {
|
||||
|
|
@ -481,12 +484,15 @@ class WebSearchTransformation:
|
|||
}
|
||||
|
||||
@staticmethod
|
||||
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
|
||||
def build_web_search_outcome_block(
|
||||
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
|
||||
) -> dict[str, object]:
|
||||
match outcome:
|
||||
case SearchSucceeded(response=response):
|
||||
return WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id=tool_use_id,
|
||||
search_response=response,
|
||||
with_snippets=with_snippets,
|
||||
)
|
||||
case SearchFailed(error_code=error_code):
|
||||
return WebSearchTransformation.build_web_search_tool_result_error_block(
|
||||
|
|
|
|||
|
|
@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
|
|||
return getattr(chunk, "usage", None)
|
||||
|
||||
|
||||
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
|
||||
|
||||
|
||||
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
|
||||
|
||||
|
||||
def _numeric(value: object) -> float | None:
|
||||
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
|
||||
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
|
||||
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
|
||||
reading litellm's bill back as if the upstream had reported it."""
|
||||
hidden: Final[object] = getattr(response, "_hidden_params", None)
|
||||
if isinstance(hidden, dict):
|
||||
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
|
||||
upstream: Final = (
|
||||
hidden_params[UPSTREAM_USAGE_COST]
|
||||
if UPSTREAM_USAGE_COST in hidden_params
|
||||
else _numeric(getattr(usage, "cost", None))
|
||||
)
|
||||
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
|
||||
setattr(usage, "cost", bill)
|
||||
|
||||
|
||||
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
|
||||
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
|
||||
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
|
||||
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
|
||||
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
|
||||
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
|
||||
return None
|
||||
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
|
||||
cost: Final = (
|
||||
_numeric(hidden[UPSTREAM_USAGE_COST])
|
||||
if hidden is not None and UPSTREAM_USAGE_COST in hidden
|
||||
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
)
|
||||
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
|
|||
Values outside the range fall back to the default and warn, so a misconfigured knob can
|
||||
neither crash the caller nor silently change the meaning of what it computes.
|
||||
"""
|
||||
value: Final = get_env_int(env_var, default)
|
||||
return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
|
||||
|
||||
|
||||
def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
|
||||
"""Float counterpart of `get_env_int_in_range`."""
|
||||
return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
|
||||
|
||||
|
||||
def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
|
||||
if minimum <= value <= maximum:
|
||||
return value
|
||||
logging.getLogger("LiteLLM").warning(
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from ..exceptions import (
|
|||
ServiceUnavailableError,
|
||||
Timeout,
|
||||
UnprocessableEntityError,
|
||||
error_response_without_credentials,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -305,6 +306,16 @@ def _litellm_proxy_response(
|
|||
)
|
||||
|
||||
|
||||
def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
|
||||
if not isinstance(response, httpx.Response):
|
||||
return response
|
||||
return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
|
||||
|
||||
|
||||
def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
|
||||
return response if response is not None and (response.headers or response.content) else None
|
||||
|
||||
|
||||
def _map_openai_exception(
|
||||
*,
|
||||
model: str,
|
||||
|
|
@ -315,7 +326,7 @@ def _map_openai_exception(
|
|||
exception_provider: str,
|
||||
extra_information: str,
|
||||
) -> None:
|
||||
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
|
||||
response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
|
||||
if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
|
||||
raise ContentPolicyViolationError(
|
||||
message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
|
||||
|
|
@ -378,6 +389,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif (
|
||||
("invalid_request_error" in error_str and "content_policy_violation" in error_str)
|
||||
|
|
@ -449,6 +461,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
body=getattr(original_exception, "body", None),
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif "Request too large" in error_str:
|
||||
raise RateLimitError(
|
||||
|
|
@ -479,6 +492,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
request=_request,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif hasattr(original_exception, "status_code"):
|
||||
if original_exception.status_code == 400:
|
||||
|
|
@ -512,6 +526,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
raise BadRequestError(
|
||||
|
|
@ -563,6 +578,7 @@ def _map_openai_exception(
|
|||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
exception_status_code=original_exception.status_code,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
raise APIError(
|
||||
|
|
@ -572,6 +588,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
request=getattr(original_exception, "request", None),
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
|
|
@ -2019,6 +2036,7 @@ def _map_azure_exception(
|
|||
exception_provider: str,
|
||||
extra_information: str,
|
||||
) -> None:
|
||||
response: Final = _without_credentials(getattr(original_exception, "response", None))
|
||||
message = get_error_message(error_obj=original_exception)
|
||||
if message is None:
|
||||
if hasattr(original_exception, "message"):
|
||||
|
|
@ -2054,7 +2072,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif "DeploymentNotFound" in error_str:
|
||||
raise NotFoundError(
|
||||
|
|
@ -2062,7 +2080,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
|
||||
error_str
|
||||
|
|
@ -2076,6 +2094,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
extra_information=extra_information,
|
||||
original_exception=original_exception,
|
||||
response=response,
|
||||
)
|
||||
elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
|
||||
helpful_message: Final = (
|
||||
|
|
@ -2094,7 +2113,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif getattr(original_exception, "status_code", None) in (None, 400) and (
|
||||
|
|
@ -2106,7 +2125,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
|
||||
|
|
@ -2115,7 +2134,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
|
||||
|
|
@ -2124,7 +2143,7 @@ def _map_azure_exception(
|
|||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif "Connection error" in error_str:
|
||||
raise APIConnectionError(
|
||||
|
|
@ -2140,7 +2159,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif original_exception.status_code == 401:
|
||||
|
|
@ -2149,7 +2168,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 404:
|
||||
raise NotFoundError(
|
||||
|
|
@ -2157,7 +2176,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 408:
|
||||
raise Timeout(
|
||||
|
|
@ -2165,6 +2184,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
llm_provider="azure",
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
raise BadRequestError(
|
||||
|
|
@ -2172,7 +2192,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
raise RateLimitError(
|
||||
|
|
@ -2180,7 +2200,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 502:
|
||||
raise BadGatewayError(
|
||||
|
|
@ -2188,7 +2208,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 503:
|
||||
raise ServiceUnavailableError(
|
||||
|
|
@ -2196,7 +2216,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 504: # gateway timeout error
|
||||
raise Timeout(
|
||||
|
|
@ -2205,6 +2225,7 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
llm_provider="azure",
|
||||
exception_status_code=original_exception.status_code,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
raise APIError(
|
||||
|
|
@ -2214,6 +2235,7 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://openai.com/"),
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
|
|
@ -2589,17 +2611,20 @@ def exception_type(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(
|
||||
_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
custom_llm_provider == "litellm_proxy"
|
||||
): # handle special case where calling litellm proxy + exception str contains error message
|
||||
extract_and_raise_litellm_exception(
|
||||
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
|
||||
response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
|
||||
error_str=error_str,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
body=getattr(original_exception, "body", None),
|
||||
body=getattr(mappable_exception, "body", None),
|
||||
)
|
||||
if (
|
||||
custom_llm_provider == "openai"
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ Helper functions for health check calls.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
|
@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
|
|||
)
|
||||
|
||||
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
|
||||
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
|
||||
WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
|
||||
BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
|
||||
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
|
||||
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
|
||||
)
|
||||
|
|
@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
|
|||
return native(model, custom_llm_provider)
|
||||
|
||||
|
||||
def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
try:
|
||||
model_info: Final = ProviderConfigManager.get_provider_model_info(
|
||||
model=None, provider=LlmProviders(custom_llm_provider)
|
||||
)
|
||||
return None if model_info is None else model_info.get_base_model(model)
|
||||
except (ValueError, NotImplementedError):
|
||||
return None
|
||||
|
||||
|
||||
def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
|
||||
if custom_llm_provider == "bedrock":
|
||||
from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
|
||||
|
||||
return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
|
||||
base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
|
||||
return base_model is not None and not base_model.startswith(f"{segment}/")
|
||||
|
||||
|
||||
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
|
||||
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
provider_prefix: Final = f"{custom_llm_provider}/"
|
||||
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
|
||||
route_segments: Final = tuple(
|
||||
itertools.takewhile(
|
||||
lambda segment_index: _is_route_segment(
|
||||
custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
|
||||
),
|
||||
range(len(segments) - 1),
|
||||
)
|
||||
)
|
||||
route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
|
||||
model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
|
||||
ranked_model_ids: Final = tuple(
|
||||
(ranked_model.model.removeprefix(provider_prefix), ranked_model)
|
||||
for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
|
||||
)
|
||||
served_model_ids: Final = sorted(
|
||||
(ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
|
||||
key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
|
||||
)
|
||||
served_probes: Final = itertools.islice(
|
||||
dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
|
||||
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
|
||||
)
|
||||
provider_probes: Final = (
|
||||
f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
|
||||
)
|
||||
return tuple(dict.fromkeys((*served_probes, *provider_probes)))
|
||||
|
||||
|
||||
class HealthCheckHelpers:
|
||||
@staticmethod
|
||||
async def ahealth_check_wildcard_models(
|
||||
|
|
@ -99,23 +160,16 @@ class HealthCheckHelpers:
|
|||
litellm_logging_obj: "Logging",
|
||||
) -> dict:
|
||||
from litellm import acompletion
|
||||
from litellm.litellm_core_utils.llm_request_utils import (
|
||||
pick_cheapest_chat_models_from_llm_provider,
|
||||
)
|
||||
|
||||
# this is a wildcard model, we need to pick a random model from the provider
|
||||
cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
|
||||
if len(cheapest_models) == 0:
|
||||
probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
|
||||
if len(probe_models) == 0:
|
||||
raise Exception(
|
||||
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
||||
f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
|
||||
"no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
|
||||
)
|
||||
if len(cheapest_models) > 1:
|
||||
fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list
|
||||
else:
|
||||
fallback_models = None
|
||||
model_params["model"] = cheapest_models[0]
|
||||
model_params["model"] = probe_models[0]
|
||||
model_params["litellm_logging_obj"] = litellm_logging_obj
|
||||
model_params["fallbacks"] = fallback_models
|
||||
model_params["fallbacks"] = list(probe_models[1:]) or None
|
||||
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
|
||||
await acompletion(**model_params)
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import datetime
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
|
|
@ -151,19 +156,52 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
Returns:
|
||||
list[str]: A list of the n cheapest chat models.
|
||||
"""
|
||||
if custom_llm_provider not in litellm.models_by_provider:
|
||||
return []
|
||||
return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RankedChatModel:
|
||||
model: str
|
||||
route_rank: int
|
||||
cost: float
|
||||
is_past_deprecation_date: bool
|
||||
|
||||
|
||||
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
ranked_rows: Final = (
|
||||
(route_rank, cost, model)
|
||||
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
|
||||
ranked_models: Final = (
|
||||
RankedChatModel(
|
||||
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
|
||||
)
|
||||
for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider)
|
||||
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
|
||||
)
|
||||
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
|
||||
return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost)))
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
|
||||
class _CostMapDeprecation(TypedDict):
|
||||
deprecation_date: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
|
||||
|
||||
|
||||
def _is_past_deprecation_date(cost_map_key: str) -> bool:
|
||||
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
).get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return (
|
||||
datetime.date.fromisoformat(deprecation_date)
|
||||
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
for model in models:
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
|
@ -171,7 +209,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
|
|||
continue
|
||||
if model_info.get("mode") != "chat":
|
||||
continue
|
||||
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
yield (
|
||||
model,
|
||||
(model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0),
|
||||
_is_past_deprecation_date(model_info["key"]),
|
||||
)
|
||||
|
||||
|
||||
def _route_rank(model: str, custom_llm_provider: str) -> int | None:
|
||||
|
|
|
|||
|
|
@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
|
|||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if current_content_type and current_content_type not in [
|
||||
if media_type and media_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return current_content_type
|
||||
return media_type
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
|
|
|
|||
|
|
@ -912,9 +912,9 @@ def create_anthropic_image_param(
|
|||
image_url_input: str | dict,
|
||||
format: str | None = None,
|
||||
is_bedrock_invoke: bool = False,
|
||||
) -> AnthropicMessagesImageParam:
|
||||
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
|
||||
"""
|
||||
Create an AnthropicMessagesImageParam from an image URL input.
|
||||
Create an Anthropic content block from an image URL input.
|
||||
|
||||
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
|
||||
"""
|
||||
|
|
@ -933,14 +933,7 @@ def create_anthropic_image_param(
|
|||
if is_bedrock_invoke or image_url.startswith("http://"):
|
||||
base64_url: Final = convert_url_to_base64(url=image_url)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
else:
|
||||
# HTTPS URL - pass directly for regular Anthropic
|
||||
return AnthropicMessagesImageParam(
|
||||
|
|
@ -953,14 +946,7 @@ def create_anthropic_image_param(
|
|||
else:
|
||||
# Convert to base64 for data URIs or other formats
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
|
||||
|
||||
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
|
||||
|
|
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
|
|||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
|
||||
anthropic_content_list.append(
|
||||
cast(
|
||||
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
|
||||
_anthropic_image_param,
|
||||
)
|
||||
)
|
||||
elif content["type"] == "tool_reference":
|
||||
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
|
||||
elif content["type"] == "file":
|
||||
|
|
@ -5878,38 +5869,36 @@ def function_call_prompt(
|
|||
]
|
||||
|
||||
|
||||
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
|
||||
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
|
||||
"""
|
||||
Decides if a user-defined custom prompt or default needs to be used
|
||||
|
||||
Returns the prompt str that's passed to the model as a user message
|
||||
"""
|
||||
custom_prompt_details: dict | None = None
|
||||
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
|
||||
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict[
|
||||
f"{model}/response_schema_prompt"
|
||||
] # allow user to define custom response schema prompt by model
|
||||
elif "response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
|
||||
|
||||
if custom_prompt_details is not None:
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
|
||||
)
|
||||
else:
|
||||
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
|
||||
model_prompt_key: Final = f"{model}/response_schema_prompt"
|
||||
custom_prompt_details: Final = prompts.get(
|
||||
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
|
||||
)
|
||||
if custom_prompt_details is None:
|
||||
return default_response_schema_prompt(response_schema=response_schema)
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details,
|
||||
messages=[{"role": "user", "content": f"{response_schema}"}],
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def default_response_schema_prompt(response_schema: dict) -> str:
|
||||
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Used if provider/model doesn't support 'response_schema' param.
|
||||
|
||||
This is the default prompt. Allow user to override this with a custom_prompt.
|
||||
"""
|
||||
prompt_str: Final = f"""Use this JSON schema:
|
||||
```json
|
||||
{response_schema}
|
||||
```json
|
||||
{json.dumps(response_schema, default=str)}
|
||||
```"""
|
||||
return prompt_str
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
|
||||
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
|
||||
from litellm.llms.base_llm.realtime.transformation import (
|
||||
BaseRealtimeConfig,
|
||||
RealtimeBackend,
|
||||
RealtimeSessionUpdateRejection,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
OpenAIRealtimeEvents,
|
||||
OpenAIRealtimeOutputItemDone,
|
||||
|
|
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_MERGE_MAX_DEPTH: Final = 8
|
||||
|
||||
|
||||
class _SessionUpdateFrame(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
session: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
def _merge_session_fields(
|
||||
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
|
||||
) -> Mapping[str, object]:
|
||||
def merged_value(key: str) -> object:
|
||||
if key not in update:
|
||||
return base[key]
|
||||
value: Final = update[key]
|
||||
base_value: Final = base.get(key)
|
||||
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
|
||||
return _merge_session_fields(
|
||||
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
|
||||
)
|
||||
return update[key]
|
||||
|
||||
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
|
||||
|
||||
|
||||
def _session_update_frame(session: Mapping[str, object]) -> str:
|
||||
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
|
||||
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
|
||||
|
||||
|
||||
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
|
||||
code="session_update_too_large",
|
||||
message=(
|
||||
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
|
||||
"so this session.update was not applied"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
|
||||
|
||||
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "audio_input_not_screenable",
|
||||
"message": (
|
||||
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
|
||||
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
|
||||
),
|
||||
"param": None,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _decode_json_object(payload: str) -> Mapping[str, object]:
|
||||
"""Decode a realtime frame into its top-level field mapping."""
|
||||
return json.loads(payload)
|
||||
|
|
@ -243,8 +309,17 @@ class RealTimeStreaming:
|
|||
event_normalizer: RealtimeEventNormalizer | None = None,
|
||||
usage_meter: RealtimeUsageMeter | None = None,
|
||||
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
|
||||
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.websocket: _ClientWebSocket = websocket
|
||||
self._clock: Final = clock
|
||||
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
|
||||
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
|
||||
self._held_client_content: tuple[str, ...] = ()
|
||||
self._held_client_content_bytes: int = 0
|
||||
self._held_client_content_deadline: float | None = None
|
||||
self._replay_scope: Final = client_websocket_scope(websocket)
|
||||
self.backend_ws = backend_ws
|
||||
self.logging_obj = logging_obj
|
||||
|
|
@ -300,6 +375,10 @@ class RealTimeStreaming:
|
|||
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
|
||||
# content (realtimeInput / clientContent / toolResponse) has been sent.
|
||||
self._content_sent_after_setup: bool = False
|
||||
self._held_session_update: Mapping[str, object] | None = None
|
||||
self._applied_session_fields: Mapping[str, object] | None = None
|
||||
self._held_session_update_bytes: int = 0
|
||||
self._unscreened_audio_refused: bool = False
|
||||
# Whether this is a transcription-only session (session.type == "transcription",
|
||||
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
|
||||
# their input_audio_transcription.completed usage drives duration-based cost.
|
||||
|
|
@ -337,6 +416,17 @@ class RealTimeStreaming:
|
|||
]
|
||||
)
|
||||
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
|
||||
_CLIENT_CONTENT_TYPES = frozenset(
|
||||
[
|
||||
"conversation.item.create",
|
||||
"response.create",
|
||||
"input_audio_buffer.append",
|
||||
"input_audio_buffer.commit",
|
||||
"input_audio_buffer.end",
|
||||
]
|
||||
)
|
||||
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
|
||||
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
|
||||
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
|
||||
"pcm16": {"type": "audio/pcm", "rate": 24000},
|
||||
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
|
||||
|
|
@ -377,7 +467,7 @@ class RealTimeStreaming:
|
|||
message_obj: dict[str, Any] = cast(dict[str, Any], message)
|
||||
else:
|
||||
message_obj = cast(dict[str, Any], json.loads(message))
|
||||
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
|
||||
self._collect_tool_calls_from_response_done(message_obj)
|
||||
if not self._should_store_message(message_obj):
|
||||
return
|
||||
try:
|
||||
|
|
@ -502,7 +592,7 @@ class RealTimeStreaming:
|
|||
self.store_message(flush_event)
|
||||
self._capture_transcription_usage(flush_event)
|
||||
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
|
||||
"""Extract function_call items from response.done events for spend logging."""
|
||||
try:
|
||||
if event_obj.get("type") != "response.done":
|
||||
|
|
@ -607,6 +697,167 @@ class RealTimeStreaming:
|
|||
self._usage_meter.observe_client_event(sent_event)
|
||||
return True
|
||||
|
||||
def _builds_setup_lazily(self) -> bool:
|
||||
if self.provider_config is None:
|
||||
return False
|
||||
return self.provider_config.builds_setup_from_session_update()
|
||||
|
||||
def _holds_session_updates_until_first_content(self) -> bool:
|
||||
return self.session_configuration_request is None and self._builds_setup_lazily()
|
||||
|
||||
async def _rejected_setup_session_update(self, message: str) -> bool:
|
||||
provider_config: Final = self.provider_config
|
||||
if provider_config is None or not provider_config.builds_setup_from_session_update():
|
||||
return False
|
||||
client_event: Final = _decode_json_object(message)
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
if self._leaves_applied_setup_unchanged(update):
|
||||
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
|
||||
return True
|
||||
rejection: Final = provider_config.setup_session_update_rejection(
|
||||
self.model, update, setup_sent=self.session_configuration_request is not None
|
||||
)
|
||||
if rejection is None:
|
||||
return False
|
||||
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
|
||||
return True
|
||||
|
||||
def _held_session_update_message(self) -> str:
|
||||
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
|
||||
|
||||
async def _hold_session_update(self, message: str) -> None:
|
||||
client_event: Final = _decode_json_object(message)
|
||||
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
|
||||
)
|
||||
return
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
|
||||
self._held_session_update = held
|
||||
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
|
||||
await self._acknowledge_session_setup(held)
|
||||
|
||||
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
|
||||
if self.provider_config is None:
|
||||
return None
|
||||
preview: Final = self.provider_config.transform_realtime_request(
|
||||
_session_update_frame(session), self.model, None
|
||||
)
|
||||
return next(
|
||||
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
|
||||
)
|
||||
|
||||
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
|
||||
applied: Final = self._applied_session_fields
|
||||
if self.session_configuration_request is None or applied is None:
|
||||
return False
|
||||
applied_setup: Final = self._setup_preview(applied)
|
||||
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
|
||||
return (
|
||||
applied_setup is not None
|
||||
and merged_setup is not None
|
||||
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
|
||||
)
|
||||
|
||||
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
|
||||
if self.provider_config is None:
|
||||
return
|
||||
setup: Final = self._setup_preview(session)
|
||||
if setup is None:
|
||||
return
|
||||
acknowledged: Final = self.provider_config.transform_session_created_event(
|
||||
self.model, str(self.logging_obj.litellm_trace_id), setup
|
||||
)
|
||||
if acknowledged is None:
|
||||
return
|
||||
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
|
||||
event_str: Final = json.dumps(event)
|
||||
self.store_message(event_str)
|
||||
await self._send_event_to_client(event, event_str)
|
||||
|
||||
async def _send_held_session_update_as_setup(self) -> None:
|
||||
held: Final = self._held_session_update
|
||||
if held is None:
|
||||
return
|
||||
message: Final = self._held_session_update_message()
|
||||
self._held_session_update = None
|
||||
self._held_session_update_bytes = 0
|
||||
if await self._send_to_backend(message):
|
||||
self._applied_session_fields = held
|
||||
await self._send_held_client_content()
|
||||
|
||||
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
|
||||
held_types: Final = (
|
||||
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
|
||||
if self._uses_deferred_backend_setup()
|
||||
else RealTimeStreaming._HELD_CONTENT_TYPES
|
||||
)
|
||||
return (
|
||||
msg_type in held_types
|
||||
and self._held_session_update is not None
|
||||
and self._holds_session_updates_until_first_content()
|
||||
and not self._pending_messages_until_setup
|
||||
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
|
||||
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
|
||||
)
|
||||
|
||||
def _hold_client_content(self, message: str) -> None:
|
||||
if self._held_client_content_deadline is None:
|
||||
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
|
||||
self._held_client_content = (*self._held_client_content, message)
|
||||
self._held_client_content_bytes += len(message.encode("utf-8"))
|
||||
|
||||
async def _send_held_client_content(self) -> None:
|
||||
held: Final = self._held_client_content
|
||||
self._held_client_content = ()
|
||||
self._held_client_content_bytes = 0
|
||||
self._held_client_content_deadline = None
|
||||
for message in held:
|
||||
await self._send_to_backend(message)
|
||||
|
||||
def _setup_deadline(self) -> float:
|
||||
content_deadline: Final = self._held_client_content_deadline
|
||||
return (
|
||||
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
|
||||
)
|
||||
|
||||
async def _send_setup_before_content(self, msg_type: str | None) -> None:
|
||||
if self._holds_session_updates_until_first_content():
|
||||
await self._send_held_session_update_as_setup()
|
||||
await self._send_default_setup_before_first_client_message(msg_type)
|
||||
|
||||
async def _receive_client_message_before_setup_deadline(self) -> str | None:
|
||||
if not self._holds_session_updates_until_first_content():
|
||||
return await self._receive_client_message()
|
||||
receive: Final = asyncio.ensure_future(self._receive_client_message())
|
||||
try:
|
||||
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
|
||||
if not done:
|
||||
await self._send_setup_before_content(None)
|
||||
return await receive
|
||||
finally:
|
||||
receive.cancel()
|
||||
|
||||
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
|
||||
if self.session_configuration_request is not None or self.provider_config is None:
|
||||
return
|
||||
if msg_type == "session.update" or (
|
||||
self._uses_deferred_backend_setup()
|
||||
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
|
||||
):
|
||||
return
|
||||
default_setup: Final = self.provider_config.session_configuration_request(self.model)
|
||||
if not default_setup:
|
||||
return
|
||||
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
|
||||
await self.backend_ws.send(setup)
|
||||
self._cache_session_configuration_request(setup)
|
||||
self._applied_session_fields = _NO_SESSION_FIELDS
|
||||
await self._send_held_client_content()
|
||||
|
||||
def _enforce_transcription_session_model(self, message: str) -> str:
|
||||
"""Force client transcription session updates to the authorized model.
|
||||
|
||||
|
|
@ -851,6 +1102,20 @@ class RealTimeStreaming:
|
|||
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
|
||||
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
|
||||
|
||||
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
|
||||
if pre_block_backend_message is None and self._is_transcription_session:
|
||||
return
|
||||
await self._send_setup_before_content("conversation.item.create")
|
||||
# Deliver any caller-supplied backend message FIRST so that
|
||||
# protocol contracts requiring a specific ordering (e.g.
|
||||
# Gemini Live's mandatory ``toolResponse`` after a
|
||||
# ``toolCall``) are honored before the guardrail's own
|
||||
# clientContent / cancel messages are sent.
|
||||
if pre_block_backend_message is not None:
|
||||
await self._send_to_backend(pre_block_backend_message)
|
||||
if not self._is_transcription_session:
|
||||
await self._withdraw_blocked_turn(item_id)
|
||||
|
||||
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
|
||||
await self._send_guardrail_cancel()
|
||||
if item_id is not None and self._deletes_conversation_items:
|
||||
|
|
@ -958,6 +1223,21 @@ class RealTimeStreaming:
|
|||
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
|
||||
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
|
||||
|
||||
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
|
||||
return (
|
||||
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
|
||||
and self._builds_setup_lazily()
|
||||
and self._should_disable_vad_auto_response()
|
||||
)
|
||||
|
||||
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
|
||||
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
|
||||
return
|
||||
self._unscreened_audio_refused = True
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
|
||||
)
|
||||
|
||||
async def run_realtime_guardrails(
|
||||
self,
|
||||
transcript: str,
|
||||
|
|
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
|
|||
# Use realtime_violation_message if configured; fall back to guardrail error text.
|
||||
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
|
||||
|
||||
# Deliver any caller-supplied backend message FIRST so that
|
||||
# protocol contracts requiring a specific ordering (e.g.
|
||||
# Gemini Live's mandatory ``toolResponse`` after a
|
||||
# ``toolCall``) are honored before the guardrail's own
|
||||
# clientContent / cancel messages are sent.
|
||||
if pre_block_backend_message is not None:
|
||||
await self._send_to_backend(pre_block_backend_message)
|
||||
if not self._is_transcription_session:
|
||||
await self._withdraw_blocked_turn(item_id)
|
||||
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
|
||||
# Send the policy violation hint (shows as small gray status text in UI).
|
||||
await self.websocket.send_text(
|
||||
json.dumps(
|
||||
|
|
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
|
|||
break
|
||||
finally:
|
||||
self._flushing_pending_messages_until_setup = False
|
||||
if self.session_created_sent_to_client and self._builds_setup_lazily():
|
||||
await self._maybe_send_guardrail_turn_detection_update()
|
||||
continue
|
||||
if self.session_created_sent_to_client:
|
||||
# A synthetic session.created (with placeholder defaults) was
|
||||
# already forwarded to the client when we connected. The
|
||||
|
|
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
|
|||
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
|
||||
|
||||
def _backend_refused_session(self, close: BackendClose) -> bool:
|
||||
return close.code != 1000 and not self._backend_sent_frames
|
||||
return (
|
||||
close.code != 1000
|
||||
and not self._backend_sent_frames
|
||||
and not self._holds_session_updates_until_first_content()
|
||||
)
|
||||
|
||||
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
|
||||
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
|
||||
|
|
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
|
|||
client_event: _ClientEventFrame
|
||||
try:
|
||||
while True:
|
||||
message = await self._receive_client_message()
|
||||
message = await self._receive_client_message_before_setup_deadline()
|
||||
if message is None:
|
||||
return ClientLoopExit.CLIENT_DISCONNECTED
|
||||
|
||||
|
|
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
|
|||
if (
|
||||
msg_type == "session.update"
|
||||
and self.session_configuration_request is None
|
||||
and self._held_session_update is None
|
||||
and not self._guardrail_turn_detection_update_sent
|
||||
and self._should_disable_vad_auto_response()
|
||||
and (
|
||||
|
|
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
|
|||
except (json.JSONDecodeError, AttributeError):
|
||||
pass
|
||||
|
||||
if self._refuses_unscreened_audio(msg_type):
|
||||
await self._refuse_unscreened_audio(msg_type, message)
|
||||
continue
|
||||
|
||||
## LOGGING
|
||||
# Log after any in-place modifications (GA remap, guardrail
|
||||
# turn_detection injection) so audit logs reflect what we
|
||||
# actually forward to the backend.
|
||||
self.store_input(message=message)
|
||||
|
||||
if self._holds_client_content(msg_type, message):
|
||||
self._hold_client_content(message)
|
||||
continue
|
||||
if self._held_client_content and msg_type != "session.update":
|
||||
await self._send_setup_before_content(msg_type)
|
||||
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
|
||||
await self._send_setup_before_content(msg_type)
|
||||
|
||||
if self._should_buffer_client_message_until_setup(message):
|
||||
self._buffer_pending_message_until_setup(message)
|
||||
continue
|
||||
|
|
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
|
|||
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
|
||||
if msg_type == "session.update" and self.provider_config is None:
|
||||
await self._send_session_gate_once()
|
||||
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
|
||||
continue
|
||||
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
|
||||
await self._hold_session_update(message)
|
||||
if self._pending_messages_until_setup:
|
||||
await self._send_held_session_update_as_setup()
|
||||
continue
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
|
||||
await self._send_setup_before_content(msg_type)
|
||||
try:
|
||||
sent = await self._send_to_backend(message)
|
||||
except RealtimeTranscriptionProtocolError as e:
|
||||
|
|
|
|||
|
|
@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
USAGE_COST_BILLED_PROVIDERS,
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
chunk_usage,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
|
|
@ -1380,9 +1381,6 @@ def stream_chunk_builder_text_completion(
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
|
||||
|
||||
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
if logging_obj is None:
|
||||
return None
|
||||
|
|
@ -1390,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
|||
return provider
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> float | None:
|
||||
|
|
@ -1404,9 +1398,6 @@ def _stream_builder_response_cost(
|
|||
for chunk in reversed(chunks)
|
||||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
|
||||
|
|
@ -1453,13 +1444,15 @@ def _set_stream_builder_response_cost(
|
|||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
|
||||
getattr(usage, "cost", None), (int, float)
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
|
||||
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
|
||||
return
|
||||
setattr(usage, "cost", computed_cost)
|
||||
_set_hidden_response_cost(response, float(computed_cost))
|
||||
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
|
||||
|
||||
|
||||
def _field_of(value: object, name: str) -> object:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from pydantic import TypeAdapter
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
|
|
@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesToolResultParam,
|
||||
AnthropicMessagesToolUseParam,
|
||||
AnthropicSearchResultParam,
|
||||
CompactionBlock,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -937,6 +939,8 @@ def _count_messages(
|
|||
default_token_count,
|
||||
params.pdf_pricing,
|
||||
)
|
||||
elif key == "provider_specific_fields":
|
||||
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
|
||||
elif key == "search_results" and isinstance(value, list):
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_search_results_text,
|
||||
|
|
@ -951,6 +955,25 @@ def _count_messages(
|
|||
return num_tokens
|
||||
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
|
||||
fields: Final = as_str_mapping(provider_specific_fields)
|
||||
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
block_fields: Final = (
|
||||
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
|
||||
)
|
||||
return tuple(
|
||||
content
|
||||
for block in block_fields
|
||||
if block is not None
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(content := block.get("content"), str)
|
||||
and content
|
||||
)
|
||||
|
||||
|
||||
def _count_extra(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
|
|
@ -1648,6 +1671,11 @@ def _count_anthropic_content(
|
|||
return tokens
|
||||
|
||||
|
||||
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
|
||||
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
|
||||
)
|
||||
|
||||
|
||||
def _count_content_list(
|
||||
count_function: TokenCounterFunction,
|
||||
content_list: str
|
||||
|
|
@ -1659,6 +1687,7 @@ def _count_content_list(
|
|||
| AnthropicMessagesImageParam
|
||||
| AnthropicMessagesDocumentParam
|
||||
| AnthropicSearchResultParam
|
||||
| CompactionBlock
|
||||
],
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
|
|
@ -1670,8 +1699,10 @@ def _count_content_list(
|
|||
for c in content_list:
|
||||
if isinstance(c, str):
|
||||
num_tokens += count_function(c)
|
||||
elif c["type"] == "text":
|
||||
num_tokens += count_function(str(c.get("text", "")))
|
||||
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
|
||||
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
|
||||
if block_text:
|
||||
num_tokens += count_function(block_text)
|
||||
elif c["type"] == "image_url":
|
||||
image_url = c.get("image_url")
|
||||
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
|
||||
|
|
@ -1704,12 +1735,6 @@ def _count_content_list(
|
|||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("thinking", "redacted_thinking"):
|
||||
# Claude extended thinking content block
|
||||
# Count the thinking text and skip the opaque blobs (signature, redacted data)
|
||||
thinking_text = str(c.get("thinking", ""))
|
||||
if thinking_text:
|
||||
num_tokens += count_function(thinking_text)
|
||||
elif c["type"] == "tool_reference":
|
||||
# Anthropic tool-search reference block: a lightweight pointer to
|
||||
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
|
||||
|
|
@ -1727,7 +1752,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"tool_reference, search_result)."
|
||||
f"compaction, tool_reference, search_result)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
|||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
|
|
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
unappliable_request_rewrite,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
|
|
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
|
|||
text: str
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _WritableMessage(Protocol):
|
||||
@overload
|
||||
def get(self, key: str, /) -> object | None: ...
|
||||
|
|
@ -195,6 +197,140 @@ class _ToolCallShape:
|
|||
arguments: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _WebSearchString:
|
||||
block_idx: int
|
||||
result_idx: int | None
|
||||
field: str
|
||||
value: str
|
||||
|
||||
|
||||
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
|
||||
choice: Final = response.choices[0] if response.choices else None
|
||||
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
|
||||
blocks: Final = (
|
||||
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
|
||||
if fields is not None and isinstance(fields.get("web_search_results"), list)
|
||||
else ()
|
||||
)
|
||||
return tuple(
|
||||
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
|
||||
)
|
||||
|
||||
|
||||
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
|
||||
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
|
||||
|
||||
|
||||
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
|
||||
fields: Final = as_str_mapping(result)
|
||||
if fields is None or fields.get("type") != "web_search_result":
|
||||
return ()
|
||||
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
|
||||
return tuple(
|
||||
_WebSearchString(block_idx, result_idx, field, value)
|
||||
for field, value in values
|
||||
if isinstance(value, str) and value
|
||||
)
|
||||
|
||||
|
||||
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
|
||||
block_id: Final = block.get("id")
|
||||
match block.get("type"):
|
||||
case "server_tool_use":
|
||||
return True
|
||||
case "tool_use":
|
||||
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _web_search_block_strings(
|
||||
block_idx: int, block: Mapping[str, object], *, with_query: bool
|
||||
) -> tuple[_WebSearchString, ...]:
|
||||
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
|
||||
``web_search_tool_result`` result's title and url."""
|
||||
match block.get("type"):
|
||||
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
|
||||
tool_input: Final = as_str_mapping(block.get("input"))
|
||||
query: Final = None if tool_input is None else tool_input.get("query")
|
||||
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
|
||||
case "web_search_tool_result":
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_result_strings(block_idx, result_idx, result)
|
||||
for result_idx, result in enumerate(_web_search_results(block))
|
||||
)
|
||||
)
|
||||
case _:
|
||||
return ()
|
||||
|
||||
|
||||
def _rewritten_web_search_result(
|
||||
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
|
||||
) -> object:
|
||||
fields: Final = as_str_mapping(result)
|
||||
if fields is None:
|
||||
return result
|
||||
return {
|
||||
**fields,
|
||||
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
|
||||
}
|
||||
|
||||
|
||||
def _rewritten_web_search_block(
|
||||
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
|
||||
) -> Mapping[str, object]:
|
||||
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
|
||||
if _is_server_tool_use(block):
|
||||
query: Final = by_location.get((None, "query"))
|
||||
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
|
||||
return block if query is None else {**block, "input": {**tool_input, "query": query}}
|
||||
if not isinstance(block.get("content"), list):
|
||||
return block
|
||||
return {
|
||||
**block,
|
||||
"content": [
|
||||
_rewritten_web_search_result(result_idx, result, by_location)
|
||||
for result_idx, result in enumerate(_web_search_results(block))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
|
||||
block: Final = as_str_mapping(content_block)
|
||||
if block is None or not isinstance(content_block, _WritableMessage):
|
||||
return
|
||||
rewritten: Final = _rewritten_web_search_block(block, rewrites)
|
||||
writable: Final = _as_writable(content_block)
|
||||
for key in ("input", "content"):
|
||||
if key in rewritten:
|
||||
writable[key] = rewritten[key]
|
||||
|
||||
|
||||
def _split_web_search_texts(
|
||||
guardrailed: Sequence[str], web_search_count: int, reply_count: int
|
||||
) -> tuple[Sequence[str], Sequence[str]]:
|
||||
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
|
||||
return guardrailed[:split_at], guardrailed[split_at:]
|
||||
|
||||
|
||||
def _web_search_rewrites_by_block(
|
||||
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
|
||||
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
|
||||
rewrites: Final = tuple(
|
||||
(target, value)
|
||||
for target, value in zip(strings, guardrailed)
|
||||
if isinstance(value, str) and value != target.value
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
|
||||
for block_idx in {target.block_idx for target, _ in rewrites}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
|
||||
return (
|
||||
block.get("type") == "tool_use"
|
||||
|
|
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
|
|||
|
||||
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_DECODED_SSE_DATA: Final = TypeAdapter(object)
|
||||
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
|
||||
_RELEASED_TOOL_USE_STOP: Final = (
|
||||
b"event: message_delta\n"
|
||||
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
|
||||
|
|
@ -315,6 +452,51 @@ def _write_back_tool_use(
|
|||
block["name"] = shape.name
|
||||
|
||||
|
||||
def _streamed_query_strings(
|
||||
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
|
||||
) -> tuple[_WebSearchString, ...]:
|
||||
if len(tool_call_blocks) != len(tool_calls):
|
||||
return ()
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(
|
||||
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
|
||||
)
|
||||
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
target.block_idx: value
|
||||
for target, value in zip(strings, guardrailed)
|
||||
if isinstance(value, str) and value != target.value
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
|
||||
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
|
||||
if arguments is None:
|
||||
return tool_call
|
||||
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
|
||||
|
||||
|
||||
def _with_query_rewrites(
|
||||
tool_calls: tuple[_ToolCallShape, ...],
|
||||
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
|
||||
query_rewrites: Mapping[int, str],
|
||||
) -> tuple[_ToolCallShape, ...]:
|
||||
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
|
||||
return tool_calls
|
||||
return tuple(
|
||||
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
|
||||
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SSEFieldRewrite:
|
||||
"""One field of one nested section of a buffered SSE event, rewritten."""
|
||||
|
|
@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
def post_call_hook_response(self, response: object) -> object:
|
||||
if not isinstance(response, ModelResponse):
|
||||
return response
|
||||
return self.adapter.translate_openai_response_to_anthropic(response)
|
||||
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
|
||||
result_blocks: Final = _streamed_web_search_result_blocks(response)
|
||||
if not result_blocks:
|
||||
return translated
|
||||
content: Final = translated.get("content") or ()
|
||||
return {**translated, "content": [*result_blocks, *content]}
|
||||
|
||||
@staticmethod
|
||||
def _build_streaming_usage_response(
|
||||
|
|
@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
task_mappings,
|
||||
tool_calls_to_check,
|
||||
)
|
||||
block_fields: Final = tuple(map(self._output_block_fields, response_content))
|
||||
web_search_strings: Final = tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(content_idx, fields[1], with_query=True)
|
||||
for content_idx, fields in enumerate(block_fields)
|
||||
if fields is not None
|
||||
)
|
||||
)
|
||||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
if texts_to_check or tool_calls_to_check or web_search_strings:
|
||||
request_data = self._prepare_request_data(
|
||||
request_data,
|
||||
response,
|
||||
|
|
@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
)
|
||||
|
||||
inputs: Final = self._build_guardrail_inputs(
|
||||
texts_to_check,
|
||||
[*(target.value for target in web_search_strings), *texts_to_check],
|
||||
images_to_check,
|
||||
tool_calls_to_check,
|
||||
response,
|
||||
|
|
@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
|
||||
web_search_rewrites, reply_rewrites = _split_web_search_texts(
|
||||
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
|
||||
)
|
||||
|
||||
# Step 3: Map guardrail responses back to original response structure
|
||||
await self._apply_guardrail_responses_to_output(
|
||||
response=response,
|
||||
responses=guardrailed_texts,
|
||||
responses=list(reply_rewrites),
|
||||
task_mappings=task_mappings,
|
||||
)
|
||||
self._write_back_web_search_strings(
|
||||
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
|
||||
|
||||
|
|
@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
string_so_far = first_choice.message.content
|
||||
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
|
||||
guardrail_inputs: Final = GenericGuardrailAPIInputs()
|
||||
if string_so_far:
|
||||
guardrail_inputs["texts"] = [string_so_far]
|
||||
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
|
||||
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
|
||||
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
|
||||
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
|
||||
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
|
||||
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
|
||||
if texts_to_check:
|
||||
guardrail_inputs["texts"] = texts_to_check
|
||||
if tool_calls_list:
|
||||
guardrail_inputs["tool_calls"] = tool_calls_list
|
||||
|
||||
|
|
@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
responses_so_far, request_data
|
||||
)
|
||||
raise
|
||||
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
|
||||
web_search_rewrites, reply_rewrites = _split_web_search_texts(
|
||||
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
|
||||
)
|
||||
if (
|
||||
deliver_ended_stream_rewrites
|
||||
and isinstance(string_so_far, str)
|
||||
and string_so_far
|
||||
and guardrailed_texts
|
||||
and guardrailed_texts[0] != string_so_far
|
||||
and reply_rewrites
|
||||
and reply_rewrites[0] != string_so_far
|
||||
):
|
||||
self._write_ended_stream_text_rewrite(
|
||||
responses_so_far,
|
||||
guardrailed_texts[0],
|
||||
reply_rewrites[0],
|
||||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
if deliver_ended_stream_rewrites:
|
||||
self._write_ended_stream_web_search_rewrites(
|
||||
responses_so_far,
|
||||
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
|
||||
)
|
||||
if deliver_ended_stream_rewrites:
|
||||
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
|
||||
self._write_ended_stream_tool_call_rewrites(
|
||||
responses_so_far,
|
||||
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
|
||||
post_guardrail_tool_calls=_tool_call_shapes(
|
||||
returned_tool_calls
|
||||
if isinstance(returned_tool_calls, list)
|
||||
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
|
||||
else tool_calls_list or ()
|
||||
post_guardrail_tool_calls=_with_query_rewrites(
|
||||
_tool_call_shapes(
|
||||
returned_tool_calls
|
||||
if isinstance(returned_tool_calls, list)
|
||||
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
|
||||
else tool_calls_list or ()
|
||||
),
|
||||
tool_call_blocks,
|
||||
_query_rewrites(
|
||||
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
|
||||
),
|
||||
),
|
||||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
|
|
@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
request_data["litellm_metadata"] = user_metadata
|
||||
return request_data
|
||||
|
||||
@staticmethod
|
||||
def _write_back_web_search_strings(
|
||||
response_content: Sequence[object],
|
||||
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
|
||||
) -> None:
|
||||
for block_idx, rewrites in rewrites_by_block.items():
|
||||
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
|
||||
|
||||
@staticmethod
|
||||
def _get_response_content(response: object) -> Sequence[object]:
|
||||
"""Extract content list from a dict or object response."""
|
||||
|
|
@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
) -> None:
|
||||
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
|
||||
buffered chunks in place: the rebuilt response lists tool calls in the
|
||||
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
|
||||
on the nth block, its first ``input_json_delta`` carrying the full rewritten
|
||||
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
|
||||
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
|
||||
arguments, every later one blanked, and ``content_block_start`` carrying the
|
||||
rewritten name. Blocks that do not line up with the rebuilt tool calls make
|
||||
the rewrite undeliverable, so the pipeline executor discards it and releases
|
||||
the original chunks."""
|
||||
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
|
||||
return
|
||||
block_indices: Final = tuple(
|
||||
index
|
||||
for item in responses_so_far
|
||||
for event in cls._iter_sse_events(item)
|
||||
if event.get("type") == "content_block_start"
|
||||
and isinstance(block := event.get("content_block"), Mapping)
|
||||
and block.get("type") == "tool_use"
|
||||
and isinstance(index := event.get("index"), int)
|
||||
)
|
||||
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
|
||||
if len(block_indices) != len(post_guardrail_tool_calls):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
raise UndeliverableStreamRewrite(
|
||||
guardrail_name,
|
||||
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
|
||||
f"{len(block_indices)} tool_use blocks",
|
||||
f"{len(block_indices)} tool_use or server_tool_use blocks",
|
||||
)
|
||||
rewrites_by_block: Final = MappingProxyType(
|
||||
{
|
||||
|
|
@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
|
||||
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
|
||||
result_blocks: Final = (
|
||||
(index, block)
|
||||
for event in events
|
||||
if event.get("type") == "content_block_start"
|
||||
and isinstance(index := event.get("index"), int)
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
)
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _write_ended_stream_web_search_rewrites(
|
||||
cls,
|
||||
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
|
||||
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
|
||||
) -> None:
|
||||
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
|
||||
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
|
||||
if not rewrites_by_block:
|
||||
return
|
||||
|
||||
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
|
||||
index: Final = event.get("index")
|
||||
block: Final = as_str_mapping(event.get("content_block"))
|
||||
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
|
||||
if event.get("type") != "content_block_start" or block is None or rewrites is None:
|
||||
return None
|
||||
return _SSEFieldRewrite(
|
||||
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
|
||||
)
|
||||
|
||||
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
|
||||
|
||||
@classmethod
|
||||
def _streamed_tool_call_blocks(
|
||||
cls, responses_so_far: Sequence[object]
|
||||
) -> tuple[tuple[int, Mapping[str, object]], ...]:
|
||||
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
|
||||
return tuple(
|
||||
(index, block)
|
||||
for event in events
|
||||
if event.get("type") == "content_block_start"
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
and isinstance(block_type := block.get("type"), str)
|
||||
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
|
||||
and isinstance(index := event.get("index"), int)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
|
||||
"""The id of each buffered ``tool_use`` block, by its content block index."""
|
||||
|
|
@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
|
||||
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
|
||||
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
|
||||
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
|
||||
return StreamingScanKey(
|
||||
texts=(self.get_streaming_string_so_far(responses_so_far),),
|
||||
texts=(*(target.value for target in result_strings), streamed_text),
|
||||
tool_calls=tool_use_fingerprints if stream_ended else (),
|
||||
stream_ended=stream_ended,
|
||||
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ import re
|
|||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -24,6 +25,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
|||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_JSON_VALUE: Final = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
|
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
|
|||
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
|
||||
|
||||
|
||||
def _message_blocks(message: object) -> Sequence[object]:
|
||||
fields: Final = as_str_mapping(message)
|
||||
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
|
||||
return blocks or ()
|
||||
|
||||
|
||||
def _is_signed_compaction_block(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
if fields is None or fields.get("type") != "compaction":
|
||||
return False
|
||||
signature: Final = fields.get("signature")
|
||||
return isinstance(signature, str) and bool(signature)
|
||||
|
||||
|
||||
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
|
||||
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
|
||||
)
|
||||
|
||||
|
||||
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
|
||||
if not isinstance(context_management, list):
|
||||
return context_management
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
|
||||
[entry for entry in context_management if isinstance(entry, dict)]
|
||||
)
|
||||
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
|
||||
|
||||
|
||||
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
|
||||
mapping: Final = as_str_mapping(spec)
|
||||
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
|
||||
|
||||
|
||||
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
|
||||
spec: Final = anthropic_context_management(context_management)
|
||||
return spec if _context_management_edits(spec) else None
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
|
||||
edits: Final = _context_management_edits(anthropic_context_management(context_management))
|
||||
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
any(edit_type != "compact_20260112" for edit_type in edit_types),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
|
|
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
context_management=context_management,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
|
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
verbose_logger.debug("Making request to: %s", endpoint_url)
|
||||
|
||||
# Get required headers
|
||||
headers: Final = self.get_count_tokens_headers(auth_header)
|
||||
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
|
||||
|
||||
# Use LiteLLM's async httpx client
|
||||
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
context_management=context_management.spec if context_management is not None else None,
|
||||
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
carries_signed_compaction_block,
|
||||
context_management_betas,
|
||||
context_management_with_edits,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
|
||||
"""
|
||||
|
|
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
|
|||
drop_custom=drop_params,
|
||||
),
|
||||
),
|
||||
("context_management", context_management_with_edits(context_management)),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
|
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
|
|||
)
|
||||
)
|
||||
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
|
||||
def get_count_tokens_headers(
|
||||
self,
|
||||
auth_header: Mapping[str, str],
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
context_management: JsonValue = None,
|
||||
) -> dict[str, str]:
|
||||
"""The count-tokens headers around a resolved Anthropic auth header
|
||||
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
|
||||
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
|
||||
merges with the token-counting beta instead of replacing it."""
|
||||
merges with the token-counting beta instead of replacing it. A history that carries a
|
||||
signed compaction block also needs the compaction beta, or Anthropic rejects the block
|
||||
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
|
||||
compaction_betas: Final = (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
|
||||
if carries_signed_compaction_block(messages)
|
||||
else ()
|
||||
)
|
||||
request_betas: Final = (
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
*compaction_betas,
|
||||
*context_management_betas(context_management),
|
||||
)
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
**auth_header,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
),
|
||||
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
|
||||
}
|
||||
|
||||
def validate_request(
|
||||
|
|
|
|||
|
|
@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
request_data["output_config"] = extra_kwargs["output_config"]
|
||||
|
||||
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
|
||||
api_base: Final = extra_kwargs.get("api_base")
|
||||
(
|
||||
openai_request,
|
||||
tool_name_mapping,
|
||||
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
|
||||
request_data,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
|
|||
TOOL_NAME_HASH_LENGTH: Final = 8
|
||||
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
|
||||
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
|
||||
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
|
||||
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
|
|
@ -255,7 +256,12 @@ class AnthropicAdapter:
|
|||
return result
|
||||
|
||||
def translate_completion_input_params_with_tool_mapping(
|
||||
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
|
||||
self,
|
||||
kwargs,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
drop_params: bool = True,
|
||||
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
|
||||
"""
|
||||
Translate Anthropic request params to OpenAI format, returning tool name mapping.
|
||||
|
|
@ -289,6 +295,7 @@ class AnthropicAdapter:
|
|||
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
|
||||
anthropic_message_request=request_body,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
|
@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
def _supports_prompt_cache_key(
|
||||
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
|
||||
) -> bool:
|
||||
if not model or not custom_llm_provider:
|
||||
return False
|
||||
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
|
||||
|
|
@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
supported_params: Final = litellm.get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
return "prompt_cache_key" in (supported_params or ())
|
||||
if "prompt_cache_key" not in (supported_params or ()):
|
||||
return False
|
||||
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
|
||||
return True
|
||||
_, _, _, resolved_api_base = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
)
|
||||
mistral_config: Final = litellm.MistralConfig()
|
||||
return mistral_config.takes_prompt_cache_key_value(
|
||||
prompt_cache_key
|
||||
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
|
||||
|
||||
@staticmethod
|
||||
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
|
||||
|
|
@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_kwargs: ChatCompletionRequest,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> None:
|
||||
"""Translate metadata fields from Anthropic request to OpenAI request."""
|
||||
if "metadata" in anthropic_message_request:
|
||||
|
|
@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_kwargs["user"] = metadata["user_id"]
|
||||
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
|
||||
if prompt_cache_key is not None and self._supports_prompt_cache_key(
|
||||
anthropic_message_request.get("model"), custom_llm_provider
|
||||
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
|
||||
):
|
||||
new_kwargs["prompt_cache_key"] = prompt_cache_key
|
||||
|
||||
|
|
@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
preserve_midturn_system: bool = False,
|
||||
drop_params: bool = True,
|
||||
) -> tuple[ChatCompletionRequest, dict[str, str]]:
|
||||
|
|
@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
## CONVERT TOOL CHOICE
|
||||
self._translate_tool_choice_to_openai(
|
||||
|
|
|
|||
|
|
@ -12,11 +12,20 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
||||
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
|
||||
citations: Final = block.get("citations")
|
||||
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
|
||||
|
||||
|
||||
class FakeAnthropicMessagesStreamIterator:
|
||||
"""
|
||||
|
|
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
|
||||
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
|
||||
"""Build SSE chunks for a single content block."""
|
||||
chunks: Final = []
|
||||
chunks: Final = list[bytes]()
|
||||
block_type: Final = block_dict.get("type")
|
||||
|
||||
if block_type == "text":
|
||||
citations: Final = _citations_of(block_dict)
|
||||
content_block_start = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
"content_block": {"type": "text", "text": "", "citations": ()}
|
||||
if citations
|
||||
else {"type": "text", "text": ""},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
citation_deltas: Final = (
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "citations_delta", "citation": citation},
|
||||
}
|
||||
for citation in citations
|
||||
)
|
||||
chunks.extend(
|
||||
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
|
||||
)
|
||||
text: Final = block_dict.get("text", "")
|
||||
content_block_delta = {
|
||||
"type": "content_block_delta",
|
||||
|
|
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
|
||||
|
||||
elif block_type == "redacted_thinking":
|
||||
content_block_start = {
|
||||
elif block_type in ("tool_use", "server_tool_use"):
|
||||
empty_input: Final[Mapping[str, object]] = {}
|
||||
tool_use_start: Final = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {"type": "redacted_thinking"},
|
||||
"content_block": {**block_dict, "input": empty_input},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
|
||||
elif block_type == "tool_use":
|
||||
content_block_start = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {
|
||||
"type": "tool_use",
|
||||
"id": block_dict.get("id"),
|
||||
"name": block_dict.get("name"),
|
||||
"input": {},
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
input_data: Final = block_dict.get("input", {})
|
||||
content_block_delta = {
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
|
||||
tool_use_delta: Final = {
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {
|
||||
"type": "input_json_delta",
|
||||
"partial_json": json.dumps(input_data),
|
||||
"partial_json": json.dumps(block_dict.get("input", {})),
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
|
||||
|
||||
else:
|
||||
passthrough_start: Final = {
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from ...common_utils import (
|
|||
OBJECT_LIST_ADAPTER,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
|
|
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
if transformed_context_management is not None:
|
||||
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
|
||||
elif drops_unsupported_params(litellm_params.drop_params):
|
||||
anthropic_messages_optional_request_params.pop("context_management")
|
||||
|
||||
####### get required params for all anthropic messages requests ######
|
||||
# Lazy %s: the f-string previously stringified the entire messages
|
||||
|
|
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", ())
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
# the schema under output_config.format; the older top-level
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from ..openai.common_utils import (
|
|||
build_output_token_limit_response,
|
||||
is_output_token_limit_error,
|
||||
model_dump_provider_response,
|
||||
upstream_error_response,
|
||||
)
|
||||
from .common_utils import (
|
||||
AzureOpenAIError,
|
||||
|
|
@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=error_body,
|
||||
)
|
||||
|
|
@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=error_body,
|
||||
)
|
||||
|
|
@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
error_text = error_response.text
|
||||
raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
async def make_async_azure_httpx_request(
|
||||
self,
|
||||
|
|
@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
except Exception as e:
|
||||
error_code: Final[object] = getattr(e, "status_code", None)
|
||||
if error_code is not None:
|
||||
raise AzureOpenAIError(status_code=error_code, message=str(e))
|
||||
raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e))
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
|
||||
|
||||
from ...openai.common_utils import upstream_error_response
|
||||
from ...openai.completion.transformation import OpenAITextCompletionConfig
|
||||
from ..common_utils import AzureOpenAIError, BaseAzureLLM
|
||||
|
||||
|
|
@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
||||
async def acompletion(
|
||||
self,
|
||||
|
|
@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
||||
def streaming(
|
||||
self,
|
||||
|
|
@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.exceptions import ContentPolicyViolationError
|
||||
|
||||
|
||||
|
|
@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping:
|
|||
model: str,
|
||||
extra_information: str,
|
||||
original_exception: Exception,
|
||||
response: httpx.Response | None,
|
||||
) -> ContentPolicyViolationError:
|
||||
"""
|
||||
Create a content policy violation error
|
||||
|
|
@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping:
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
provider_specific_fields={
|
||||
# Preserve legacy key for backward compatibility.
|
||||
"innererror": inner_error,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
|||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import *
|
||||
from litellm.types.responses.main import *
|
||||
|
|
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
|
||||
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
|
||||
|
||||
def deployment_names_wrap_openai_models(self) -> bool:
|
||||
return True
|
||||
|
||||
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
|
||||
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Azure Responses API does not support context_management (compaction).
|
||||
|
|
|
|||
|
|
@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
content_parts,
|
||||
convert_content_list_to_str,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
is_pdf_file_part,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
|
|
@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
|
|||
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
|
||||
from litellm.llms.xai.chat.transformation import XAIChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
|
||||
from litellm.utils import add_path_to_api_base, supports_tool_choice
|
||||
|
|
@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
|
|||
return any(_is_list_only_content_part(part) for part in content_parts(message))
|
||||
|
||||
|
||||
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
|
||||
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return ()
|
||||
try:
|
||||
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
|
||||
except ValidationError:
|
||||
return ()
|
||||
return tuple(part for part in parts if part.get("type") == "text")
|
||||
|
||||
|
||||
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
|
||||
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
|
||||
return message
|
||||
text_only: Final = {**message, "content": list(text_parts)}
|
||||
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
|
||||
|
||||
|
||||
_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
|
||||
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
|
||||
return None
|
||||
return PromptCacheBreakpoint(mode="explicit")
|
||||
|
||||
|
||||
def _part_with_foundry_breakpoint(part: object) -> object:
|
||||
if not isinstance(part, Mapping):
|
||||
return part
|
||||
fields: Final = _FIELDS.validate_python(part)
|
||||
if "prompt_cache_breakpoint" not in fields:
|
||||
return fields
|
||||
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
|
||||
|
||||
|
||||
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
|
||||
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
|
||||
parts: Final = content_parts(renamed) if drop_params else ()
|
||||
if not parts:
|
||||
return renamed
|
||||
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
|
||||
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
|
||||
|
||||
|
||||
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
|
||||
model_info: Final = litellm_params.get("model_info")
|
||||
if not isinstance(model_info, Mapping):
|
||||
return any(
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
|
||||
)
|
||||
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
|
||||
if isinstance(deployment_flag, bool):
|
||||
return deployment_flag
|
||||
return any(
|
||||
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
|
||||
for name in named_models
|
||||
)
|
||||
|
||||
|
||||
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
|
||||
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
|
||||
usage: Final = chunk.get("usage")
|
||||
|
|
@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
self,
|
||||
messages: list[AllMessageValues],
|
||||
model: str,
|
||||
reads_breakpoints: bool | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> list:
|
||||
"""
|
||||
- Azure AI Studio doesn't support content as a list. This handles:
|
||||
|
|
@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
Azure AI Foundry backends set additionalProperties=false and reject
|
||||
these with "Extra inputs are not permitted", which breaks multi-turn
|
||||
Anthropic-format clients that echo thinking blocks back as history.
|
||||
2. Transforms list content to a string.
|
||||
2. Transforms list content to a string, except on deployments that read OpenAI explicit
|
||||
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
|
||||
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
|
||||
content part can carry the breakpoint (a message-level one is silently ignored), so those
|
||||
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
|
||||
way the openai provider renames it. Every other part is dropped, exactly as the string
|
||||
conversion drops it, before the marks are renamed, so a message-level mark lands on the
|
||||
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
|
||||
shape Foundry takes, and a malformed one is dropped.
|
||||
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
|
||||
|
||||
Operates on a deep copy so the caller's messages keep their thinking blocks
|
||||
and provider metadata, which a fallback to another provider still needs.
|
||||
"""
|
||||
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
deployment_reads_breakpoints: Final = (
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
|
||||
if reads_breakpoints is None
|
||||
else reads_breakpoints
|
||||
)
|
||||
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
stripped_messages: Final = (
|
||||
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
|
||||
if deployment_reads_breakpoints
|
||||
else copied_messages
|
||||
)
|
||||
for message in stripped_messages:
|
||||
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
|
||||
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
|
||||
|
|
@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
if _has_list_only_content_part(message):
|
||||
continue
|
||||
|
||||
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
|
||||
message_dict["content"] = list(text_parts)
|
||||
continue
|
||||
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
|
|
@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
if extra_body and isinstance(extra_body, dict):
|
||||
optional_params.update(extra_body)
|
||||
optional_params.pop("max_retries", None)
|
||||
return super().transform_request(model, messages, optional_params, litellm_params, headers)
|
||||
return {
|
||||
"model": model,
|
||||
"messages": self._transform_messages(
|
||||
messages,
|
||||
model,
|
||||
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
|
||||
bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
),
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import dataclasses
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
from typing import TYPE_CHECKING, Any, Literal, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
|
||||
from litellm.types.realtime import (
|
||||
RealtimeInputAudioTranscriptionUsage,
|
||||
|
|
@ -22,6 +24,26 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class RealtimeSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
def error_event(self, client_event_id: object) -> dict[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": self.code,
|
||||
"message": self.message,
|
||||
"param": self.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class RealtimeBackend(Protocol):
|
||||
async def __aenter__(self) -> object: ...
|
||||
|
||||
|
|
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
|
|||
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
|
||||
return None
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return False
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
return None
|
||||
|
||||
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -27,6 +27,20 @@ else:
|
|||
BaseLLMException = Any
|
||||
|
||||
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
|
||||
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
|
||||
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
|
||||
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{"summary": REASONING_SUMMARY_VALUES}
|
||||
)
|
||||
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
|
||||
)
|
||||
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
|
||||
)
|
||||
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
|
||||
)
|
||||
|
||||
|
||||
class BaseResponsesAPIConfig(ABC):
|
||||
|
|
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
|
|||
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
|
||||
return "detailed"
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues | None:
|
||||
return None
|
||||
|
||||
def deployment_names_wrap_openai_models(self) -> bool:
|
||||
return False
|
||||
|
||||
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
|
||||
return None
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import redact_string, verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
|
||||
|
|
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
|
||||
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
|
||||
from litellm.types.llms.bedrock import AwsAuthParams
|
||||
from litellm.types.llms.openai import OpenAIRealtimeEvents
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from ..common_utils import BedrockError
|
||||
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
|
||||
from .transformation import BedrockRealtimeConfig, requested_modalities
|
||||
|
||||
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
|
@ -256,23 +256,11 @@ async def _ack_session_update(
|
|||
async def _reject_session_update(
|
||||
client_ws: RealtimeClientWebSocket,
|
||||
parsed_client_message: Mapping[str, JsonValue],
|
||||
rejection: BedrockSessionUpdateRejection,
|
||||
rejection: RealtimeSessionUpdateRejection,
|
||||
) -> bool:
|
||||
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
|
||||
client_event_id: Final = parsed_client_message.get("event_id")
|
||||
error_event: Final = {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": rejection.code,
|
||||
"message": rejection.message,
|
||||
"param": rejection.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
try:
|
||||
await client_ws.send_text(json.dumps(error_event))
|
||||
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
|
||||
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
|
||||
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import math
|
|||
import uuid as uuid_lib
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter
|
||||
|
|
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
|
|||
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockLockedSessionSettings:
|
||||
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
|
||||
|
|
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
|
||||
"""The error for a session.update this session cannot apply, or None when it applies."""
|
||||
session_config: Final = json_message.get("session")
|
||||
if not isinstance(session_config, Mapping):
|
||||
|
|
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
|
||||
if malformed:
|
||||
name, expected = malformed[0]
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_type",
|
||||
message=(
|
||||
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
|
||||
|
|
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
)
|
||||
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
|
||||
if unplayable_voice is not None:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
|
||||
|
|
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
if not self.prompt_started:
|
||||
return None
|
||||
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
|
||||
param="session",
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
|
|||
def custom_llm_provider(self) -> str:
|
||||
return "bigmodel"
|
||||
|
||||
@property
|
||||
def prompts_unsupported_json_schema(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
|
||||
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
|
||||
|
||||
|
|
|
|||
|
|
@ -122,6 +122,12 @@ class Authenticator:
|
|||
return tokens["access_token"]
|
||||
|
||||
def get_account_id(self, access_token: str | None = None) -> str | None:
|
||||
return self._account_id(access_token, persist_derived=True)
|
||||
|
||||
def read_account_id(self, access_token: str | None = None) -> str | None:
|
||||
return self._account_id(access_token, persist_derived=False)
|
||||
|
||||
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
|
||||
auth_data: Final = self._read_auth_file()
|
||||
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
|
||||
return extract_chatgpt_account_id(access_token)
|
||||
|
|
@ -133,7 +139,7 @@ class Authenticator:
|
|||
id_token: Final = auth_data.get("id_token")
|
||||
saved_access_token: Final = auth_data.get("access_token")
|
||||
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
|
||||
if derived:
|
||||
if derived and persist_derived:
|
||||
self._write_auth_file({**auth_data, "account_id": derived})
|
||||
return derived
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
safe_convert_created_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
|
|
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def supports_native_websocket(self) -> bool:
|
||||
"""ChatGPT does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_REASONING_FIELD_VALUES
|
||||
|
|
|
|||
|
|
@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
|
|||
),
|
||||
)
|
||||
|
||||
# Auto-send session setup if the provider requires it (e.g.
|
||||
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
|
||||
# Auto-send session setup at connect unless the provider builds it from the
|
||||
# client's first session.update (Gemini/Vertex AI Live accept exactly one
|
||||
# setup, so RealTimeStreaming sends theirs on the first client message).
|
||||
# Build the streaming handler first so a transcription guardrail's
|
||||
# auto-response disable can be folded into this one setup: Gemini
|
||||
# rejects a second setup, so a follow-up disable would be dropped
|
||||
# and the guardrail bypassed.
|
||||
_session_config: str | None = None
|
||||
if provider_config.requires_session_configuration():
|
||||
sends_setup_at_connect: Final = (
|
||||
provider_config.requires_session_configuration()
|
||||
and not provider_config.builds_setup_from_session_update()
|
||||
)
|
||||
if sends_setup_at_connect:
|
||||
_session_config = provider_config.session_configuration_request(model)
|
||||
if _session_config:
|
||||
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ from urllib.parse import unquote
|
|||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import (
|
||||
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
|
||||
ReasoningFieldValues,
|
||||
)
|
||||
from litellm.llms.fireworks_ai.common_utils import (
|
||||
FIREWORKS_FORWARD_USER_ID_PARAM,
|
||||
get_fireworks_forwarded_user_id,
|
||||
|
|
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from collections import OrderedDict
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,8 +16,9 @@ from litellm._uuid import uuid
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
OPENAI_TO_GEMINI_TTS_VOICES,
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
|
|||
|
||||
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
|
||||
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
|
||||
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
|
||||
|
||||
|
||||
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
|
||||
)
|
||||
|
||||
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
|
||||
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"achernar",
|
||||
"achird",
|
||||
"algenib",
|
||||
"algieba",
|
||||
"alnilam",
|
||||
"aoede",
|
||||
"autonoe",
|
||||
"callirrhoe",
|
||||
"charon",
|
||||
"despina",
|
||||
"enceladus",
|
||||
"erinome",
|
||||
"fenrir",
|
||||
"gacrux",
|
||||
"iapetus",
|
||||
"kore",
|
||||
"laomedeia",
|
||||
"leda",
|
||||
"orus",
|
||||
"puck",
|
||||
"pulcherrima",
|
||||
"rasalgethi",
|
||||
"sadachbia",
|
||||
"sadaltager",
|
||||
"schedar",
|
||||
"sulafat",
|
||||
"umbriel",
|
||||
"vindemiatrix",
|
||||
"zephyr",
|
||||
"zubenelgenubi",
|
||||
}
|
||||
)
|
||||
|
||||
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
|
||||
|
||||
def _is_undocumented_live_voice(voice: object) -> bool:
|
||||
return (
|
||||
isinstance(voice, str)
|
||||
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
|
||||
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
|
||||
)
|
||||
|
||||
|
||||
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
|
||||
"""Build the Gemini Live speechConfig for a client-requested voice.
|
||||
|
||||
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
|
||||
the session on an unknown voice, so they are dropped with a warning and
|
||||
the model keeps its default voice. Every other name is forwarded verbatim.
|
||||
OpenAI stock voice names have no Gemini equivalent, so they are always
|
||||
dropped, and so is any name outside Google's documented Live voices when
|
||||
``drop_undocumented`` is set, keeping the model's default voice. Every
|
||||
other name is forwarded verbatim.
|
||||
"""
|
||||
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
if not isinstance(voice, str):
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
|
||||
"dropping it so the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
if drop_undocumented and _is_undocumented_live_voice(voice):
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
|
||||
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
|
||||
|
||||
|
|
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
|
|||
return max(len(data) * 3 // 4 - padding, 0)
|
||||
|
||||
|
||||
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _chat_completion_tool(tool: object) -> dict[str, object]:
|
||||
fields: Final = as_str_mapping(tool) or {}
|
||||
if fields.get("type") != "function" or "function" in fields:
|
||||
return {**fields}
|
||||
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
|
||||
|
||||
|
||||
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
|
||||
parts: Final = system_instruction.get("parts", []) if system_instruction else []
|
||||
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
|
||||
return "\n".join(texts) if texts else None
|
||||
|
||||
|
||||
class GeminiRealtimeConfig(BaseRealtimeConfig):
|
||||
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, drop_params: bool = False):
|
||||
super().__init__()
|
||||
self.drop_params = drop_params
|
||||
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
|
||||
# Gemini Live sometimes emits usageMetadata in a standalone frame between
|
||||
# turns; buffer it here so the next response.done carries the token counts.
|
||||
self._pending_usage_metadata: dict | None = None
|
||||
self._unbilled_input_audio_bytes: int = 0
|
||||
self._manual_activity_open: bool = False
|
||||
|
||||
def is_setup_message(self, msg_obj: dict) -> bool:
|
||||
return "setup" in msg_obj
|
||||
|
|
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
|
||||
return True
|
||||
|
||||
def _rejects_undocumented_voices(self, model: str) -> bool:
|
||||
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
|
||||
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
|
||||
|
||||
@staticmethod
|
||||
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
|
||||
detail_mapping: Final = as_str_mapping(details)
|
||||
|
|
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
|
||||
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
|
||||
return ()
|
||||
self._manual_activity_open = True
|
||||
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
|
||||
return (json.dumps({"realtimeInput": activity_start}),)
|
||||
|
||||
def _end_manual_activity(self) -> tuple[str, ...]:
|
||||
if not self._manual_activity_open:
|
||||
return ()
|
||||
self._manual_activity_open = False
|
||||
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
|
||||
return (json.dumps({"realtimeInput": activity_end}),)
|
||||
|
||||
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
|
||||
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
|
||||
if self._manual_turn_detection_enabled(session_configuration_request):
|
||||
realtime_input_dict: BidiGenerateContentRealtimeInput = {
|
||||
"activityEnd": True,
|
||||
}
|
||||
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
|
||||
else:
|
||||
realtime_input_dict = {"audioStreamEnd": True}
|
||||
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
|
||||
return [json.dumps({"realtimeInput": realtime_input_dict})]
|
||||
return list(self._end_manual_activity())
|
||||
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
|
||||
return [json.dumps({"realtimeInput": audio_stream_end})]
|
||||
|
||||
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
|
||||
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
|
||||
|
|
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"voice",
|
||||
]
|
||||
|
||||
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
|
||||
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
|
||||
if "generationConfig" not in optional_params:
|
||||
optional_params["generationConfig"] = {}
|
||||
for key, value in non_default_params.items():
|
||||
|
|
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
|
||||
elif key == "temperature":
|
||||
optional_params["generationConfig"]["temperature"] = value
|
||||
elif key == "max_response_output_tokens":
|
||||
elif key == "max_response_output_tokens" and isinstance(value, int):
|
||||
optional_params["generationConfig"]["maxOutputTokens"] = value
|
||||
elif key == "modalities":
|
||||
optional_params["generationConfig"]["responseModalities"] = [
|
||||
modality.upper() for modality in cast(list[str], value)
|
||||
]
|
||||
optional_params["generationConfig"]["responseModalities"] = list(
|
||||
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
|
||||
)
|
||||
elif key == "tools":
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
|
|
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
vertex_gemini_config = VertexGeminiConfig()
|
||||
# Tools should be at the top level of setup, not inside generationConfig
|
||||
optional_params["tools"] = vertex_gemini_config.map_function(
|
||||
value=value, optional_params=optional_params
|
||||
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
|
||||
optional_params=optional_params,
|
||||
)
|
||||
elif key == "input_audio_transcription" and value is not None:
|
||||
optional_params["inputAudioTranscription"] = {}
|
||||
elif key == "turn_detection" and value is not None:
|
||||
value_typed = cast(OpenAIRealtimeTurnDetection, value)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
continue
|
||||
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
|
||||
if transformed_audio_activity_config:
|
||||
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
|
||||
automaticActivityDetection=transformed_audio_activity_config
|
||||
)
|
||||
elif key == "turn_detection":
|
||||
realtime_input_config = self._realtime_input_config(cast(object, value))
|
||||
if realtime_input_config is not None:
|
||||
optional_params["realtimeInputConfig"] = realtime_input_config
|
||||
elif key == "voice":
|
||||
speech_config = _gemini_live_speech_config(value)
|
||||
speech_config = _gemini_live_speech_config(
|
||||
value, self.drop_params and self._rejects_undocumented_voices(model)
|
||||
)
|
||||
if speech_config:
|
||||
optional_params["generationConfig"]["speechConfig"] = speech_config
|
||||
if len(optional_params["generationConfig"]) == 0:
|
||||
optional_params.pop("generationConfig")
|
||||
return optional_params
|
||||
|
||||
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
|
||||
if turn_detection is None:
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
|
||||
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
return None
|
||||
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
|
||||
if not transformed_audio_activity_config:
|
||||
return None
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
|
||||
|
||||
@staticmethod
|
||||
def _extract_turn_detection(session: dict) -> dict | None:
|
||||
"""Extract turn_detection from a session.update payload.
|
||||
|
|
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
if isinstance(input_cfg, dict):
|
||||
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
|
||||
normalized["input_audio_transcription"] = input_cfg["transcription"]
|
||||
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
|
||||
normalized["turn_detection"] = input_cfg["turn_detection"]
|
||||
output_cfg: Final = audio.get("output")
|
||||
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
|
||||
normalized["voice"] = output_cfg["voice"]
|
||||
|
|
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _model_cost_entry(model: str) -> dict:
|
||||
def _model_cost_entry(model: str) -> Mapping[str, object]:
|
||||
entry = litellm.model_cost.get(model)
|
||||
if entry is None:
|
||||
stripped: Final = model.split("/", 1)[-1]
|
||||
|
|
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
|
||||
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
|
||||
normalized: Final = tuple(
|
||||
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
|
||||
def _requested_response_modalities(value: object) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value.upper(),)
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return ()
|
||||
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
|
||||
|
||||
@staticmethod
|
||||
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
|
||||
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if unknown:
|
||||
return unknown
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
|
||||
return ("TEXT",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
|
||||
return ("AUDIO",)
|
||||
return ()
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
if self.drop_params:
|
||||
return None
|
||||
if setup_sent:
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message=(
|
||||
"Gemini Live accepts one session configuration per connection, and this session already sent "
|
||||
"its configuration, so this session.update was not applied. Put every setting in session.update "
|
||||
"before the first conversation content, or set drop_params to drop it instead"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
|
||||
|
||||
@staticmethod
|
||||
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
|
||||
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
|
||||
)
|
||||
if not unsupported:
|
||||
return None
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.{param}': {model} cannot produce "
|
||||
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
|
||||
),
|
||||
param=f"session.{param}",
|
||||
)
|
||||
|
||||
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
|
||||
output_voice: Final = audio_output.get("voice")
|
||||
voice: Final = output_voice or session.get("voice")
|
||||
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
|
||||
return None
|
||||
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
|
||||
"Set drop_params to drop it instead"
|
||||
),
|
||||
param=param,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
|
||||
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
|
||||
text-only ones such as transcribe-live), then keep a single modality
|
||||
because Gemini Live rejects a setup naming two."""
|
||||
return GeminiRealtimeConfig._single_response_modality(
|
||||
GeminiRealtimeConfig._swap_unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
normalized: Final = tuple(str(modality).upper() for modality in modalities)
|
||||
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if len(known) < len(normalized):
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
|
||||
model,
|
||||
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
|
||||
)
|
||||
return known or (GeminiRealtimeConfig._default_response_modality(model),)
|
||||
|
||||
@staticmethod
|
||||
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
|
||||
model,
|
||||
)
|
||||
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
|
||||
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if len(modalities) <= 1:
|
||||
return modalities
|
||||
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
|
||||
chosen,
|
||||
", ".join(modalities),
|
||||
)
|
||||
return (chosen,)
|
||||
|
||||
@staticmethod
|
||||
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
|
||||
generation_config: Final = setup.get("generationConfig")
|
||||
|
|
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# would be silently dropped because ``map_openai_params`` only
|
||||
# recognises the flat OpenAI-beta key names.
|
||||
session_payload = self._normalize_session_payload_for_mapping(session_payload)
|
||||
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
|
||||
new_overrides: Final = self.map_openai_params(
|
||||
optional_params={}, non_default_params=session_payload, model=model
|
||||
)
|
||||
|
||||
if session_configuration_request is None:
|
||||
generation_config: Final = new_overrides.setdefault("generationConfig", {})
|
||||
|
|
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
message_str = str(message)
|
||||
raise ValueError(f"Invalid JSON message: {message_str}")
|
||||
|
||||
messages: Final[list[str]] = []
|
||||
msg_type: Final = json_message.get("type")
|
||||
|
||||
if msg_type == "session.update":
|
||||
return self._handle_session_update(json_message, model, session_configuration_request)
|
||||
|
||||
if msg_type == "response.create":
|
||||
return [] # Gemini responds automatically; nothing to forward
|
||||
return list(self._end_manual_activity())
|
||||
|
||||
if msg_type == "conversation.item.create":
|
||||
return self._handle_conversation_item(json_message)
|
||||
|
|
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
|
||||
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
|
||||
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
|
||||
messages.append(gemini_msg)
|
||||
return messages
|
||||
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
|
||||
|
||||
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
|
||||
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
|
||||
|
|
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
logging_session_id: str,
|
||||
session_configuration_request: str | None = None,
|
||||
) -> OpenAIRealtimeStreamSessionEvents:
|
||||
if session_configuration_request:
|
||||
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
|
||||
else:
|
||||
session_configuration_request_dict = {}
|
||||
session_configuration_request_dict: Final = _parse_setup(
|
||||
session_configuration_request or self.session_configuration_request(model)
|
||||
)
|
||||
|
||||
_model: Final = session_configuration_request_dict.get("model") or model
|
||||
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
|
||||
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
|
||||
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
|
||||
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
|
||||
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
|
||||
session: Final = OpenAIRealtimeStreamSession(
|
||||
id=logging_session_id,
|
||||
modalities=_modalities,
|
||||
)
|
||||
if _system_instruction is not None and isinstance(_system_instruction, str):
|
||||
session["instructions"] = _system_instruction
|
||||
if instructions is not None:
|
||||
session["instructions"] = instructions
|
||||
if _model is not None and isinstance(_model, str):
|
||||
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
|
||||
if "/models/" in _model:
|
||||
|
|
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
|
||||
return not litellm.gemini_live_defer_setup
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return True
|
||||
|
||||
def session_configuration_request(self, model: str) -> str:
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseInputParam,
|
||||
|
|
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_native_websocket(self) -> bool:
|
||||
"""GitHub Copilot does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.LITELLM_PROXY
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return ANY_REASONING_FIELD_VALUES
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/
|
|||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from itertools import accumulate, count
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
|
|
@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
|
||||
from litellm.types.llms.mistral import (
|
||||
MistralCacheKeyParams,
|
||||
MistralFilePart,
|
||||
MistralReasoningEffortParams,
|
||||
MistralRequestRouting,
|
||||
MistralTextBlock,
|
||||
MistralThinkingBlock,
|
||||
MistralToolCallMessage,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
from litellm.utils import convert_to_model_response_object, supports_reasoning
|
||||
|
|
@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
|
|||
|
||||
|
||||
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
|
||||
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
|
||||
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
|
||||
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
|
||||
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
|
||||
|
||||
|
||||
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
|
||||
requested: Final = as_str_mapping(reasoning_effort)
|
||||
if requested is None:
|
||||
return reasoning_effort, None
|
||||
return requested.get("effort"), requested.get("summary")
|
||||
|
||||
|
||||
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
|
||||
|
|
@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
|
||||
"""
|
||||
|
||||
_is_base_class = False
|
||||
|
||||
temperature: int | None = None
|
||||
top_p: int | None = None
|
||||
max_tokens: int | None = None
|
||||
|
|
@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
is_magistral: Final = "magistral" in model.lower()
|
||||
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
|
||||
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
|
||||
return [
|
||||
"stream",
|
||||
"temperature",
|
||||
|
|
@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
"parallel_tool_calls",
|
||||
*(("thinking",) if is_magistral else ()),
|
||||
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
|
||||
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
|
||||
]
|
||||
|
||||
def _map_tool_choice(self, tool_choice: str) -> str:
|
||||
|
|
@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
optional_params["extra_body"] = {"random_seed": value}
|
||||
if param == "response_format":
|
||||
optional_params["response_format"] = value
|
||||
if param == "reasoning_effort" and "magistral" not in model.lower():
|
||||
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
|
||||
if (
|
||||
param == "reasoning_effort"
|
||||
and "magistral" not in model.lower()
|
||||
and (
|
||||
reasoning_level := self._reasoning_level_to_send(
|
||||
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
|
||||
)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
optional_params["reasoning_effort"] = reasoning_level
|
||||
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
|
||||
# Flag that we need to add reasoning system prompt
|
||||
optional_params["_add_reasoning_prompt"] = True
|
||||
if param == "parallel_tool_calls":
|
||||
optional_params["parallel_tool_calls"] = value
|
||||
if param in ("parallel_tool_calls", "prompt_cache_key"):
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def _reasoning_level_to_send(
|
||||
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
|
||||
) -> str | None:
|
||||
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
|
||||
sendable: Final = (
|
||||
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
|
||||
)
|
||||
if sendable is not None and summary is None:
|
||||
return sendable
|
||||
if drop_params or litellm.drop_params is True:
|
||||
return sendable
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
|
||||
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
|
||||
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
llm_provider=self.custom_llm_provider,
|
||||
model=model,
|
||||
)
|
||||
|
||||
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
|
||||
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
|
||||
hostname: Final = urlparse(resolved_api_base).hostname or ""
|
||||
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
|
||||
|
||||
@staticmethod
|
||||
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
|
||||
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
|
||||
|
||||
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
|
||||
if not self.accepts_prompt_cache_key_at(api_base):
|
||||
return "since only the Mistral API (*.mistral.ai) takes it"
|
||||
if not self.takes_prompt_cache_key_value(prompt_cache_key):
|
||||
return "since the Mistral API takes it only as a string of ASCII characters"
|
||||
return None
|
||||
|
||||
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
|
||||
requested_api_base: Final = routing.get("api_base")
|
||||
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
|
||||
allowed_openai_params: Final = routing.get("allowed_openai_params")
|
||||
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
|
||||
return False
|
||||
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
|
||||
if rejection is None:
|
||||
return False
|
||||
if litellm.drop_params is True or routing.get("drop_params") is True:
|
||||
return True
|
||||
requested_provider: Final = routing.get("custom_llm_provider")
|
||||
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
|
||||
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
|
||||
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
llm_provider=provider,
|
||||
model=model,
|
||||
)
|
||||
|
||||
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
|
||||
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
|
||||
api_base = (
|
||||
|
|
@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
messages = self._handle_message_with_file(messages)
|
||||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
file_part: Final = block.get("file") if block.get("type") == "file" else None
|
||||
file_id: Final = (
|
||||
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
|
||||
)
|
||||
if not file_id:
|
||||
return block
|
||||
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
|
||||
|
||||
@classmethod
|
||||
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return message
|
||||
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
|
||||
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
|
||||
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
|
||||
)
|
||||
|
||||
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
|
||||
"""
|
||||
Mistral API supports only 'file_id' in message content with type 'file'.
|
||||
"""
|
||||
for m in messages:
|
||||
_content_block = m.get("content")
|
||||
if _content_block and isinstance(_content_block, list):
|
||||
if any(c.get("type") == "file" for c in _content_block):
|
||||
# If file content is present, we get file_id from 'file' attribute of content block
|
||||
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
|
||||
file_contents = [c for c in _content_block if c.get("type") == "file"]
|
||||
for file_content in file_contents:
|
||||
file_id = file_content.get("file", {}).get("file_id")
|
||||
if file_id:
|
||||
# Replace 'file' with 'file_id'
|
||||
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
|
||||
file_content.pop("file", None)
|
||||
return messages
|
||||
return [self._with_mistral_file_ids(m) for m in messages]
|
||||
|
||||
def _add_reasoning_system_prompt_if_needed(
|
||||
self, messages: list[AllMessageValues], optional_params: dict
|
||||
|
|
@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
|
||||
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
|
||||
|
||||
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
|
||||
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
|
||||
withheld_params: Final = (
|
||||
("client_metadata", "prompt_cache_key")
|
||||
if "prompt_cache_key" in cache_key_params
|
||||
and self._prompt_cache_key_withheld(
|
||||
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
|
||||
)
|
||||
else ("client_metadata",)
|
||||
)
|
||||
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
|
||||
|
||||
# Call parent transform_request which handles _transform_messages
|
||||
return super().transform_request(
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ if TYPE_CHECKING:
|
|||
from aiohttp import ClientSession
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
|
|
@ -148,6 +149,14 @@ class OpenAIError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
def upstream_error_response(error: Exception) -> httpx.Response | None:
|
||||
if not isinstance(error, (openai.APIStatusError, httpx.HTTPStatusError)):
|
||||
return None
|
||||
return error_response_without_credentials(
|
||||
error.response.status_code, error.response, "https://docs.litellm.ai/docs"
|
||||
)
|
||||
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
|
|||
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..common_utils import BaseOpenAILLM, OpenAIError
|
||||
from ..common_utils import BaseOpenAILLM, OpenAIError, upstream_error_response
|
||||
from ..workload_identity import build_async_openai_client, build_openai_client
|
||||
from .transformation import OpenAITextCompletionConfig
|
||||
|
||||
|
|
@ -164,7 +164,12 @@ class OpenAITextCompletion(BaseLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
async def acompletion(
|
||||
self,
|
||||
|
|
@ -220,7 +225,12 @@ class OpenAITextCompletion(BaseLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
def streaming(
|
||||
self,
|
||||
|
|
@ -261,7 +271,12 @@ class OpenAITextCompletion(BaseLLM):
|
|||
error_response: object = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=response,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from litellm.utils import ProviderConfigManager
|
|||
|
||||
from ...base_llm.image_variations.transformation import BaseImageVariationConfig
|
||||
from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, upstream_error_response
|
||||
from ..workload_identity import build_async_openai_client, build_openai_client
|
||||
|
||||
|
||||
|
|
@ -129,7 +129,12 @@ class OpenAIImageVariationsHandler:
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
def image_variations(
|
||||
self,
|
||||
|
|
@ -245,4 +250,9 @@ class OpenAIImageVariationsHandler:
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import litellm
|
|||
from litellm import LlmProviders
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -64,6 +65,7 @@ from .common_utils import (
|
|||
is_openai_backed_api_base,
|
||||
is_output_token_limit_error,
|
||||
model_dump_provider_response,
|
||||
upstream_error_response,
|
||||
)
|
||||
from .workload_identity import (
|
||||
build_async_openai_client,
|
||||
|
|
@ -970,8 +972,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
litellm.remove_index_from_tool_calls(messages=messages)
|
||||
else:
|
||||
raise e
|
||||
except OpenAIError as e:
|
||||
raise e
|
||||
except (OpenAIError, UnsupportedParamsError):
|
||||
raise
|
||||
except Exception as e:
|
||||
status_code: Final = getattr(e, "status_code", 500)
|
||||
error_headers = getattr(e, "headers", None)
|
||||
|
|
@ -983,6 +985,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=error_body,
|
||||
)
|
||||
|
|
@ -1110,6 +1113,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=exception_body,
|
||||
)
|
||||
|
|
@ -1282,6 +1286,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
raise OpenAIError(
|
||||
status_code=getattr(e, "status_code", 500),
|
||||
message=str(e),
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=exception_body,
|
||||
)
|
||||
|
|
@ -1409,7 +1414,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
def embedding(
|
||||
self,
|
||||
|
|
@ -1499,7 +1509,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
async def aimage_generation(
|
||||
self,
|
||||
|
|
@ -1666,7 +1681,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
original_response=str(e),
|
||||
)
|
||||
if hasattr(e, "status_code"):
|
||||
raise OpenAIError(status_code=getattr(e, "status_code", 500), message=str(e))
|
||||
raise OpenAIError(
|
||||
status_code=getattr(e, "status_code", 500),
|
||||
message=str(e),
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
else:
|
||||
raise OpenAIError(status_code=500, message=str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ The only provider quirks:
|
|||
Ref: https://docs.perplexity.ai/api-reference/responses-post
|
||||
"""
|
||||
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
|
||||
|
|
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
|
||||
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
|
||||
)
|
||||
|
||||
|
||||
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
||||
|
|
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
|||
def supports_native_websocket(self) -> bool:
|
||||
"""Perplexity does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return PERPLEXITY_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from ..common_utils import AWSEventStreamDecoder, SagemakerError
|
||||
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
inference_component_name: Final = optional_params.get("model_id")
|
||||
if not isinstance(inference_component_name, str):
|
||||
return headers
|
||||
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
|
||||
other_headers: Final = {
|
||||
name: value
|
||||
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
|
||||
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
|
||||
}
|
||||
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
|
|
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
container_request: Final = {
|
||||
key: value for key, value in request.items() if key not in self.aws_authentication_params
|
||||
}
|
||||
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
|
||||
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
|
||||
served_model_name: Final = litellm_params.get("hf_model_name")
|
||||
if not isinstance(served_model_name, str):
|
||||
return container_request
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
|
|||
from litellm import verbose_logger
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
from litellm.types.utils import StreamingChatCompletionChunk
|
||||
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
|
||||
|
||||
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
|
||||
|
|
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
|
|||
return _load_sagemaker_response_stream_shape()
|
||||
|
||||
|
||||
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
|
||||
|
||||
|
||||
_SAGEMAKER_PROVIDERS: Final = frozenset(
|
||||
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
|
||||
)
|
||||
|
||||
|
||||
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
|
||||
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
|
||||
if isinstance(custom_llm_provider, str) and custom_llm_provider:
|
||||
return custom_llm_provider
|
||||
model: Final = litellm_params.get("model")
|
||||
return model.split("/", 1)[0] if isinstance(model, str) else None
|
||||
|
||||
|
||||
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
|
||||
headers: Final = litellm_params.get("extra_headers")
|
||||
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
value
|
||||
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
|
||||
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def with_inference_component_hint(error_message: str) -> str:
|
||||
if "Inference Component Name header is required" in error_message:
|
||||
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse, Usage
|
||||
from litellm.utils import token_counter
|
||||
|
||||
from ..common_utils import SagemakerError
|
||||
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
inference_params: Final = optional_params.copy()
|
||||
inference_params: Final = {
|
||||
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
|
||||
}
|
||||
stream: Final = inference_params.pop("stream", False)
|
||||
data: Final[dict] = {"parameters": inference_params}
|
||||
if stream is True:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue