mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_aiand_brand_drive
# Conflicts: # litellm/litellm_core_utils/llm_request_utils.py
This commit is contained in:
commit
95ad02f344
317 changed files with 19122 additions and 5247 deletions
10
.github/workflows/cost-map-guard.yml
vendored
10
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -8,6 +8,7 @@ on:
|
|||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
|
|
@ -21,16 +22,19 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
- name: Check out the base branch tip and fetch the pull request head and its merge base
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
git checkout -q --detach "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
|
|
|
|||
152
.github/workflows/lens-install-smoke.yml
vendored
152
.github/workflows/lens-install-smoke.yml
vendored
|
|
@ -1,152 +0,0 @@
|
|||
name: Lens installation smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
lens_image:
|
||||
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
lens_upgrade_image:
|
||||
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
gateway_baseline_ref:
|
||||
description: Previously qualified gateway source commit, full SHA
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lens-install-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
- component: ui
|
||||
dockerfile: ui/Dockerfile
|
||||
- component: migrations
|
||||
dockerfile: migrations/Dockerfile
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
baseline: true
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
baseline: true
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
baseline: true
|
||||
env:
|
||||
COMPONENT: ${{ matrix.component }}
|
||||
DOCKERFILE: ${{ matrix.dockerfile }}
|
||||
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
|
||||
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
|
||||
CURRENT_REF: ${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.SOURCE_REF }}
|
||||
- name: Verify the selected source
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
|
||||
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
|
||||
test "$SOURCE_REF" != "$CURRENT_REF"
|
||||
fi
|
||||
- name: Build the matching release image
|
||||
run: |
|
||||
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
|
||||
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
|
||||
- name: Save the matching release image
|
||||
run: |
|
||||
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
|
||||
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
|
||||
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
|
||||
compression-level: 0
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
|
||||
helm-install:
|
||||
needs: build-images
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: lens-install-*-${{ github.sha }}
|
||||
merge-multiple: true
|
||||
path: ${{ runner.temp }}/lens-install-images
|
||||
- name: Load the matching release images
|
||||
run: |
|
||||
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
|
||||
gzip -dc "$archive" | docker load
|
||||
rm "$archive"
|
||||
done
|
||||
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
|
||||
with:
|
||||
cosign-release: v2.4.3
|
||||
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ vars.LENS_READ_USER || github.actor }}
|
||||
password: ${{ secrets.LENS_READ_TOKEN }}
|
||||
- name: Verify independent Lens images before loading them
|
||||
env:
|
||||
LENS_IMAGE: ${{ inputs.lens_image }}
|
||||
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
|
||||
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
|
||||
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
|
||||
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
|
||||
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
|
||||
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
|
||||
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
|
||||
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
|
||||
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
|
||||
docker pull "$image"
|
||||
done
|
||||
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
|
||||
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
|
||||
- name: Install pinned Kubernetes test tools
|
||||
run: |
|
||||
curl --fail --location --output "$RUNNER_TEMP/kind" \
|
||||
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
|
||||
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kind"
|
||||
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
|
||||
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
|
||||
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kubectl"
|
||||
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
|
||||
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
|
||||
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
|
||||
- name: Install, ingest, upgrade, and restart both charts
|
||||
run: bash tests/e2e/migrations/lens_helm_smoke.sh
|
||||
6
.github/workflows/spend-logs-guard.yml
vendored
6
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -19,17 +19,17 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
|
|
|
|||
25
.github/workflows/test-linting.yml
vendored
25
.github/workflows/test-linting.yml
vendored
|
|
@ -48,11 +48,13 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$MERGE_BASE"
|
||||
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
|
||||
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
|
||||
|
|
@ -264,6 +266,7 @@ jobs:
|
|||
name: ui
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
|
|
@ -281,16 +284,12 @@ jobs:
|
|||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
|
|
@ -630,6 +629,12 @@ jobs:
|
|||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_no_none_string_error_fields
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
|
||||
|
||||
- name: test_check_no_none_string_error_fields
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
|
|
|
|||
9
.github/workflows/test-unit.yml
vendored
9
.github/workflows/test-unit.yml
vendored
|
|
@ -245,6 +245,7 @@ jobs:
|
|||
tests/unit/llms/chatgpt
|
||||
tests/unit/llms/clarifai
|
||||
tests/unit/llms/claude_code
|
||||
tests/unit/llms/clf_ai_gateway
|
||||
tests/unit/llms/cloudflare
|
||||
tests/unit/llms/codestral
|
||||
tests/unit/llms/codex
|
||||
|
|
@ -1150,18 +1151,20 @@ jobs:
|
|||
env:
|
||||
CI: "true"
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
|
||||
|
||||
if [ -z "$BASE_SHA" ]; then
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "Push to $GITHUB_REF_NAME: running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
changed_files=()
|
||||
|
|
|
|||
|
|
@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -5,8 +5,5 @@ dependencies:
|
|||
- name: redis
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
version: 18.19.1
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
|
||||
generated: '2026-10-09T04:18:10.643228000Z'
|
||||
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
|
||||
generated: '2026-10-30T02:13:11.505726000Z'
|
||||
|
|
|
|||
|
|
@ -39,6 +39,3 @@ dependencies:
|
|||
version: "18.19.1"
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
condition: redis.enabled
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -256,9 +256,7 @@ itself.
|
|||
|
||||
## Connect Lens
|
||||
|
||||
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
## Shutdown accounting
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
|
|||
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -402,95 +398,28 @@ shutdown drain window.
|
|||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.labels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
|
||||
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
{{- if .Values.ingress.enabled -}}
|
||||
{{- $fullName := include "litellm.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
{{- $lensMode := include "litellm.lens.mode" . -}}
|
||||
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
|
||||
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
|
||||
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
|
||||
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
|
||||
|
|
@ -46,18 +44,18 @@ spec:
|
|||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
|
||||
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
name: {{ $.Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ $.Values.lensWorker.service.port }}
|
||||
number: {{ $.Values.lens.service.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
servicePort: {{ $.Values.lensWorker.service.port }}
|
||||
serviceName: {{ $.Values.lens.service.name }}
|
||||
servicePort: {{ $.Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .paths }}
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
- ingress.yaml
|
||||
tests:
|
||||
- it: connects the proxy to an external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service ahead of the host paths
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[0].path
|
||||
value: /lens-ingest
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- ingress.yaml
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects deployment.yaml to the shared Lens service
|
||||
template: deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.hosts:
|
||||
- host: gateway.example
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in deployment.yaml
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
|
@ -668,70 +668,15 @@ serviceMonitor:
|
|||
# - test-namespace
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
dependencies:
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
|
||||
generated: '2026-10-09T04:18:10.642706000Z'
|
||||
|
|
@ -6,8 +6,3 @@ version: 0.1.0
|
|||
appVersion: "0.1.0"
|
||||
annotations:
|
||||
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
|
||||
|
||||
dependencies:
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
# LiteLLM componentized chart
|
||||
|
||||
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
|
||||
This chart deploys separate gateway, backend and UI services
|
||||
|
||||
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
|
|||
collector containers through an emptyDir. Empty when the sidecar is off
|
||||
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
|
||||
*/}}
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -635,93 +631,27 @@ shutdown drain window.
|
|||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -166,14 +166,14 @@ spec:
|
|||
- host: {{ $componentHosts.gateway | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
|
|
@ -249,15 +249,15 @@ spec:
|
|||
port:
|
||||
number: {{ $gatewayPort }}
|
||||
{{- end }}
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- /*
|
||||
--- Operator-supplied extra paths (ingress.extraPaths) ---
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
|
|
@ -15,7 +15,6 @@ tests:
|
|||
ingress.componentHosts.ui: ui.example.com
|
||||
ingress.componentHosts.backend: backend.example.com
|
||||
ingress.componentHosts.gateway: gateway.example.com
|
||||
lensWorker.mode: disabled
|
||||
ingress.tls:
|
||||
- hosts:
|
||||
- ui.example.com
|
||||
|
|
@ -83,12 +82,12 @@ tests:
|
|||
ui.backendUrl: https://backend.example.com
|
||||
ui.publicUrl: https://ui.example.com
|
||||
backend.gatewayUrl: https://gateway.example.com
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.externalUrl: https://lens.example.com
|
||||
lensWorker.publicUrl: https://lens.example.com
|
||||
lensWorker.serviceTokenSecret.name: existing-lens-signing
|
||||
lensWorker.gateway.secretName: existing-lens-signing
|
||||
lens.enabled: true
|
||||
lens.service.name: existing-lens
|
||||
lens.url: http://existing-lens:4318
|
||||
lens.publicUrl: https://gateway.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: existing-lens-signing
|
||||
lens.gatewaySecret.name: existing-lens-signing
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[2].http.paths[0].path
|
||||
|
|
|
|||
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: connects the gateway to an external Lens
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: connects the backend to an external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service on the single ingress host
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: keeps the ingestion endpoint on an independently deployed Lens service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects gateway/deployment.yaml to the shared Lens service
|
||||
template: gateway/deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: connects backend/deployment.yaml to the shared Lens service
|
||||
template: backend/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: omits Lens connection settings when disabled in backend/deployment.yaml
|
||||
template: backend/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: gateway/deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,164 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
suite: Independent Lens release and credentials
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
tests:
|
||||
- it: selects a Lens release independently of the gateway
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
lensWorker.image.tag: 2.3.4
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:2.3.4
|
||||
- it: retains the pinned Lens chart default when only the gateway changes
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:0.1.0-dev.0
|
||||
- it: selects development publications explicitly
|
||||
set:
|
||||
lensWorker.image.repository: ghcr.io/berriai/lens-dev
|
||||
lensWorker.image.tag: sha-0123456789abcdef
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
|
||||
- it: preserves a verified registry mirror and digest
|
||||
set:
|
||||
lensWorker.image.repository: registry.example/lens
|
||||
lensWorker.image.tag: replaced-label
|
||||
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
- it: rejects a malformed digest
|
||||
set:
|
||||
lensWorker.image.digest: sha256:invalid
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
|
||||
- it: keeps the legacy enablement switch working
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments: {count: 0}
|
||||
- it: keeps isolated local analysis with delegated gateway access
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: LENS_MODE, value: standalone}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].emptyDir
|
||||
value: {medium: Memory, sizeLimit: 1Gi}
|
||||
|
|
@ -654,70 +654,15 @@ ui:
|
|||
topologySpreadConstraints: []
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -238,7 +238,8 @@ impl AnthropicBeta {
|
|||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
| Self::ThinkingBindingControls20260801
|
||||
| Self::ThinkingDisplayUpdates20260818 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -275,7 +276,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
BetaProvider::AzureAi,
|
||||
|
|
|
|||
|
|
@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
|
|||
baseten_models: Set[str] = set()
|
||||
fireworks_ai_embedding_models: Set = set()
|
||||
deepinfra_models: Set = set()
|
||||
clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets
|
||||
perplexity_models: Set = set()
|
||||
watsonx_models: Set = set()
|
||||
gemini_models: Set = set()
|
||||
|
|
@ -849,6 +850,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
bedrock_converse_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepinfra":
|
||||
deepinfra_models.add(key)
|
||||
elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map
|
||||
clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key
|
||||
elif value.get("litellm_provider") == "perplexity":
|
||||
perplexity_models.add(key)
|
||||
elif value.get("litellm_provider") == "watsonx":
|
||||
|
|
@ -859,6 +862,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "together_ai":
|
||||
if _is_past_deprecation_date(value):
|
||||
together_ai_models.discard(key)
|
||||
elif not key.startswith("together-ai-"):
|
||||
together_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "baseten":
|
||||
if _is_past_deprecation_date(value):
|
||||
baseten_models.discard(key)
|
||||
|
|
@ -1103,6 +1111,7 @@ model_list = list(
|
|||
| set(ollama_models)
|
||||
| bedrock_models
|
||||
| deepinfra_models
|
||||
| clf_ai_gateway_models
|
||||
| perplexity_models
|
||||
| set(maritalk_models)
|
||||
| runwayml_models
|
||||
|
|
@ -1228,6 +1237,7 @@ def _build_models_by_provider() -> dict:
|
|||
"ollama": ollama_models,
|
||||
"ollama_chat": ollama_models,
|
||||
"deepinfra": deepinfra_models,
|
||||
"clf_ai_gateway": clf_ai_gateway_models,
|
||||
"perplexity": perplexity_models,
|
||||
"maritalk": maritalk_models,
|
||||
"watsonx": watsonx_models,
|
||||
|
|
@ -2104,6 +2114,9 @@ if TYPE_CHECKING:
|
|||
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
|
||||
)
|
||||
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
|
||||
from .llms.clf_ai_gateway.chat.transformation import (
|
||||
ClfAiGatewayConfig as _ClfAiGatewayConfig,
|
||||
)
|
||||
from .llms.llamafile.chat.transformation import (
|
||||
LlamafileChatConfig as _LlamafileChatConfig,
|
||||
)
|
||||
|
|
@ -2133,6 +2146,7 @@ if TYPE_CHECKING:
|
|||
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
|
||||
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
|
||||
DeepInfraConfig: Type[_DeepInfraConfig]
|
||||
ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
|
||||
LlamafileChatConfig: Type[_LlamafileChatConfig]
|
||||
LMStudioChatConfig: Type[_LMStudioChatConfig]
|
||||
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
|
||||
|
|
|
|||
|
|
@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"MistralEmbeddingConfig",
|
||||
"OpenAIImageVariationConfig",
|
||||
"DeepInfraConfig",
|
||||
"ClfAiGatewayConfig",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
"TopazImageVariationConfig",
|
||||
"OpenAITextCompletionConfig",
|
||||
|
|
@ -975,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OpenAIImageVariationConfig",
|
||||
),
|
||||
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
|
||||
"ClfAiGatewayConfig": (
|
||||
".llms.clf_ai_gateway.chat.transformation",
|
||||
"ClfAiGatewayConfig",
|
||||
),
|
||||
"DeepgramAudioTranscriptionConfig": (
|
||||
".llms.deepgram.audio_transcription.transformation",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from contextlib import contextmanager
|
|||
from enum import Enum
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_service_target, service_target
|
||||
|
|
@ -43,6 +43,9 @@ from .s3_cache import S3Cache
|
|||
RESPONSE_CACHE_TARGET: Final = "llm_response"
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
|
||||
_DECODED_LITERAL: Final = TypeAdapter(object)
|
||||
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
|
||||
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -207,7 +210,7 @@ class Cache:
|
|||
if not redis_startup_nodes:
|
||||
_env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
|
||||
if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
|
||||
redis_startup_nodes = json.loads(_env_cluster_nodes)
|
||||
redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
|
||||
|
||||
if redis_startup_nodes:
|
||||
# Only pass GCP parameters if they are provided
|
||||
|
|
|
|||
|
|
@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
|
||||
self._structured_output: Final = structured_output
|
||||
self._first_message_item_done: bool = False
|
||||
self._reasoning_summary_streamed: bool = False
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: Union[str, "BaseModel"]
|
||||
|
|
@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
|
||||
replays_openai_history: bool = False,
|
||||
structured_output: bool = False,
|
||||
reasoning_summary_streamed: bool = False,
|
||||
) -> "ModelResponseStream":
|
||||
"""
|
||||
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
|
||||
|
|
@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
|
||||
replays_openai_history: Record the completed output's item sequence for the next turn
|
||||
structured_output: Only the first message item became content
|
||||
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
|
||||
|
||||
Returns:
|
||||
ModelResponseStream: OpenAI-formatted streaming chunk
|
||||
|
|
@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
else:
|
||||
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
|
||||
elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
|
||||
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
|
||||
elif event_type == "response.reasoning_summary_text.delta":
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=cast(int, parsed_chunk.get("summary_index")),
|
||||
index=0,
|
||||
delta=Delta(reasoning_content=content_part),
|
||||
)
|
||||
]
|
||||
|
|
@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map=self._tool_call_index_map,
|
||||
replays_openai_history=self._replays_openai_history,
|
||||
structured_output=self._structured_output,
|
||||
reasoning_summary_streamed=self._reasoning_summary_streamed,
|
||||
)
|
||||
)
|
||||
if self._structured_output and _is_message_item_done(chunk):
|
||||
self._first_message_item_done = True
|
||||
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
|
||||
self._reasoning_summary_streamed = True
|
||||
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
|
||||
|
||||
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import Final, Literal
|
|||
|
||||
from litellm.litellm_core_utils.env_utils import (
|
||||
get_env_float,
|
||||
get_env_float_in_range,
|
||||
get_env_float_or_none,
|
||||
get_env_int,
|
||||
get_env_int_in_range,
|
||||
|
|
@ -149,6 +150,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
|
|||
"DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
|
||||
) # default cooldown a deployment if 50% of requests fail in a given minute
|
||||
DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
|
||||
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
|
||||
"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
|
||||
)
|
||||
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
|
||||
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
|
||||
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
|
||||
|
|
@ -416,6 +420,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
|
|||
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
|
||||
)
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
|
||||
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
|
||||
|
||||
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
|
@ -753,7 +759,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
|
|||
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
|
||||
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
|
||||
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
|
||||
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
|
||||
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
|
||||
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
|
||||
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
|
||||
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
||||
"low": 1,
|
||||
|
|
@ -813,6 +825,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"ollama",
|
||||
"ollama_chat",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"mistral",
|
||||
"groq",
|
||||
|
|
@ -1012,6 +1025,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"api.perplexity.ai",
|
||||
"api.endpoints.anyscale.com/v1",
|
||||
"api.deepinfra.com/v1/openai",
|
||||
"api.clfaigateway.dev/v1",
|
||||
"api.mistral.ai/v1",
|
||||
"codestral.mistral.ai/v1/chat/completions",
|
||||
"codestral.mistral.ai/v1/fim/completions",
|
||||
|
|
@ -1083,6 +1097,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"deepseek",
|
||||
"tencent",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"xinference",
|
||||
"xai",
|
||||
|
|
@ -1261,7 +1276,7 @@ empower_models: Final = set(
|
|||
]
|
||||
)
|
||||
|
||||
together_ai_models: Final[set] = set(
|
||||
together_ai_models: Final[set[str]] = set(
|
||||
[
|
||||
# llama llms - chat
|
||||
"togethercomputer/llama-2-70b-chat",
|
||||
|
|
|
|||
|
|
@ -153,7 +153,9 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
cached_cost_map_key,
|
||||
cached_get_model_info_helper,
|
||||
mapped_model_fill_missing,
|
||||
model_info_with_pricing_overrides,
|
||||
runtime_model_cost_registrations,
|
||||
token_counter,
|
||||
|
|
@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
|
|||
"""
|
||||
1. If custom pricing is true, return received model name
|
||||
2. If base_model is set (e.g. for azure models), return that
|
||||
3. If completion response has model set return that
|
||||
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
|
||||
4. Check if model is passed in return that
|
||||
"""
|
||||
|
||||
return_model: str | None = None
|
||||
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
completion_response_model: str | None = None
|
||||
if completion_response is not None:
|
||||
if isinstance(completion_response, BaseModel):
|
||||
completion_response_model = getattr(completion_response, "model", None)
|
||||
elif isinstance(completion_response, dict):
|
||||
completion_response_model = completion_response.get("model", None)
|
||||
completion_response_model: Final = _get_response_model(completion_response)
|
||||
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
|
||||
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
|
||||
requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
configured_model: Final = model or (
|
||||
completion_response_model if isinstance(completion_response_model, str) else None
|
||||
|
|
@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
|
|||
else:
|
||||
return_model = model
|
||||
|
||||
elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
|
||||
elif base_model is not None or bedrock_pricing_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model
|
||||
|
||||
elif provider_response_model is not None:
|
||||
return _served_or_priced_requested_model(
|
||||
provider_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
elif completion_response_model is None and hidden_params is not None:
|
||||
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
|
||||
return_model = hidden_params.get("model", model)
|
||||
|
||||
if return_model is None and completion_response_model is not None:
|
||||
return_model = completion_response_model
|
||||
return _served_or_priced_requested_model(
|
||||
completion_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
if return_model is None and model is not None:
|
||||
return_model = model
|
||||
|
||||
if (
|
||||
return_model is not None
|
||||
and custom_llm_provider is not None
|
||||
and not _model_contains_known_llm_provider(return_model)
|
||||
): # add provider prefix if not already present, to match model_cost
|
||||
provider_prefix: Final = (
|
||||
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
|
||||
)
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
|
||||
return return_model
|
||||
if return_model is None:
|
||||
return None
|
||||
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
|
||||
|
||||
|
||||
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
|
||||
|
||||
|
||||
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
|
||||
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
|
||||
return model
|
||||
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
|
||||
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
|
||||
|
||||
|
||||
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
|
||||
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
|
||||
|
||||
|
||||
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
key: Final = cached_cost_map_key(model, custom_llm_provider)
|
||||
if key is None:
|
||||
return False
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
if entry is None:
|
||||
return _rule_declares_rates(model, custom_llm_provider)
|
||||
fill_missing: Final = mapped_model_fill_missing(key, entry)
|
||||
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
|
||||
|
||||
|
||||
def _served_or_priced_requested_model(
|
||||
served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
|
||||
) -> str:
|
||||
served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
|
||||
if requested_model is None:
|
||||
return served
|
||||
requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
|
||||
if served in (requested, requested.partition("/")[2]):
|
||||
return served
|
||||
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
|
||||
if _resolves_to_rates(served, pricing_provider):
|
||||
return served
|
||||
return requested if _resolves_to_rates(requested, pricing_provider) else served
|
||||
|
||||
|
||||
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
|
||||
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
|
||||
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
|
||||
|
|
@ -1509,6 +1544,12 @@ def _response_time_ms_for_cost(
|
|||
return litellm_logging_obj.get_response_ms()
|
||||
|
||||
|
||||
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
|
||||
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
|
||||
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
|
||||
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -1526,9 +1567,9 @@ def _adjusted_provider_billed_cost(
|
|||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=None,
|
||||
completion_tokens_cost_usd_dollar=None,
|
||||
cost_for_built_in_tools_cost_usd_dollar=None,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=adjusted_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
discount_percent=discount_percent,
|
||||
|
|
@ -2341,6 +2382,15 @@ def response_cost_calculator(
|
|||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=provider_response_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=provider_response_cost,
|
||||
original_cost=provider_response_cost,
|
||||
)
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
@ -3516,7 +3566,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
|
|||
lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
|
||||
session that should bill nothing fell through to the public rates instead.
|
||||
"""
|
||||
resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
entries: Final = (
|
||||
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
|
||||
litellm.model_cost.get(model_name),
|
||||
|
|
@ -3712,9 +3762,9 @@ def handle_realtime_transcription_cost_calculation(
|
|||
return 0.0
|
||||
|
||||
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
|
||||
model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
override_info: Final = (
|
||||
_get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
)
|
||||
|
||||
total_cost = 0.0
|
||||
|
|
@ -3745,13 +3795,6 @@ def _get_transcription_model_name_from_results(
|
|||
return None
|
||||
|
||||
|
||||
def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
|
||||
try:
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
## LiteLLM versions of the OpenAI Exception Types
|
||||
|
||||
import datetime
|
||||
import enum
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
|
@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _upstream_body(response: object) -> bytes:
|
||||
if not isinstance(response, httpx.Response):
|
||||
return b""
|
||||
try:
|
||||
return response.content
|
||||
except httpx.ResponseNotRead:
|
||||
return b""
|
||||
|
||||
|
||||
def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
|
||||
if not isinstance(upstream, httpx.Response):
|
||||
return httpx.Request(method="POST", url=fallback_url)
|
||||
try:
|
||||
upstream_request: Final = upstream.request
|
||||
except RuntimeError:
|
||||
return httpx.Request(method="POST", url=fallback_url)
|
||||
return httpx.Request(
|
||||
method=upstream_request.method,
|
||||
url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
|
||||
)
|
||||
|
||||
|
||||
def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
|
||||
if not isinstance(upstream, httpx.Response):
|
||||
return None
|
||||
try:
|
||||
return upstream.elapsed
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
|
||||
def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
|
||||
response: Final = httpx.Response(
|
||||
status_code=status_code,
|
||||
stream=httpx.ByteStream(_upstream_body(upstream)),
|
||||
request=_request_without_credentials(upstream, fallback_url),
|
||||
)
|
||||
_ = response.read()
|
||||
response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
|
||||
elapsed: Final = _upstream_elapsed(upstream)
|
||||
if elapsed is not None:
|
||||
response.elapsed = elapsed
|
||||
return response
|
||||
|
||||
|
||||
_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
|
||||
|
||||
|
||||
_MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
|
||||
|
||||
|
||||
|
|
@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
|
|||
# headers stay reachable on `e.response.headers` for callers that
|
||||
# explicitly want them; only the proxy-supplied `headers=` kwarg
|
||||
# makes it onto `self.headers`.
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
|
||||
# Mirrors FastAPI HTTPException.detail so the same instance can be
|
||||
# serialized through both the ProxyException and HTTPException paths.
|
||||
self.detail = detail if detail is not None else self.message
|
||||
self.response = httpx.Response(
|
||||
status_code=429,
|
||||
headers=_response_headers,
|
||||
content=response.content if response is not None else None,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message, response=self.response, body=None
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
|
@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
|
||||
self.response = httpx.Response(
|
||||
status_code=self.status_code,
|
||||
headers=_response_headers,
|
||||
request=httpx.Request(
|
||||
method="POST",
|
||||
url=" https://cloud.google.com/vertex-ai/",
|
||||
),
|
||||
)
|
||||
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
|
||||
super().__init__(
|
||||
self.message,
|
||||
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
|
||||
|
|
@ -908,6 +924,7 @@ class APIError(openai.APIError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: object | None = None,
|
||||
response: httpx.Response | None = None,
|
||||
):
|
||||
self.status_code = status_code
|
||||
self.message = f"litellm.APIError: {message}"
|
||||
|
|
@ -916,6 +933,8 @@ class APIError(openai.APIError):
|
|||
self.litellm_debug_info = litellm_debug_info
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
if response is not None:
|
||||
self.response = response
|
||||
if request is None:
|
||||
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
|
||||
super().__init__(self.message, request=request, body=body)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class ImageEditRequestUtils:
|
|||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
drop_params: bool | None = None,
|
||||
additional_drop_params: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Get optional parameters for the image edit API.
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.integrations.prompt_management_base import PromptManagementClient
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
|
|||
|
||||
|
||||
def _hosted_openai_dialect_flag(
|
||||
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
resolve_provider: Callable[[str], str | None],
|
||||
model_cost: Mapping[str, Mapping[str, object]] | None = None,
|
||||
) -> bool | None:
|
||||
"""
|
||||
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
|
||||
|
|
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
|
|||
"""
|
||||
import litellm
|
||||
|
||||
exact_entry: Final = litellm.model_cost.get(model)
|
||||
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
|
||||
cost_map: Final = (
|
||||
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
|
||||
if model_cost is None
|
||||
else model_cost
|
||||
)
|
||||
exact_entry: Final = cost_map.get(model)
|
||||
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
|
||||
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
|
||||
return None
|
||||
provider: Final = custom_llm_provider or resolve_provider(model)
|
||||
if provider is None or provider == "openai":
|
||||
return None
|
||||
if isinstance(exact_entry, dict) and exact_provider == provider:
|
||||
if isinstance(exact_entry, Mapping) and exact_provider == provider:
|
||||
return _hosted_entry_flag(exact_entry, lambda: provider)
|
||||
from litellm.utils import get_potential_model_names
|
||||
|
||||
|
|
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
|
|||
names["stripped_model_name"],
|
||||
names["provider_prefixed_model_name"],
|
||||
)
|
||||
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
|
||||
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
|
||||
entries: Final = (cost_map.get(candidate) for candidate in candidates)
|
||||
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
|
||||
return next((flag for flag in flags if flag is not None), None)
|
||||
|
||||
|
||||
|
|
@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return False
|
||||
return prompt_cache_options is not None or targets_openai_api(api_base)
|
||||
|
||||
@staticmethod
|
||||
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
|
||||
return (
|
||||
_hosted_openai_dialect_flag(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
AnthropicCacheControlHook._resolve_provider,
|
||||
GetModelCostMap.loaded_model_cost_map(),
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
return _served_provider_for_model(model)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
|
|||
import urllib.parse
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
class _PromptVersionResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
|
||||
|
||||
data: dict[str, object] | None = None
|
||||
|
||||
|
||||
def _sanitize_id(identifier: str) -> str:
|
||||
"""Reject path traversal characters and URL-encode the identifier."""
|
||||
if any(c in identifier for c in ("/", "\\", "#", "?")):
|
||||
|
|
@ -71,8 +79,7 @@ class ArizePhoenixClient:
|
|||
response: Final = self.http_handler.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
data: Final = response.json()
|
||||
return data.get("data")
|
||||
return _PromptVersionResponse.model_validate(response.json()).data
|
||||
|
||||
except Exception as e:
|
||||
# Check if it's an HTTP error
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
if not isinstance(metadata, dict):
|
||||
return []
|
||||
return metadata.get("guardrails") or []
|
||||
|
||||
|
||||
class CustomGuardrail(CustomLogger):
|
||||
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
|
||||
use_native_during_call_hook: ClassVar[bool] = False
|
||||
|
|
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
|
|||
return True
|
||||
raise
|
||||
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
"""
|
||||
Returns the guardrail(s) to be run from the metadata or root
|
||||
Returns the guardrail(s) to be run from the root, else from both metadata fields together
|
||||
"""
|
||||
|
||||
if "guardrails" in data:
|
||||
return data["guardrails"]
|
||||
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
|
||||
# writes to "metadata"; for thread/assistant endpoints it writes to
|
||||
# "litellm_metadata". We check the one that actually contains the "guardrails"
|
||||
# key so that a non-empty litellm_metadata without guardrails does not shadow
|
||||
# the merged list stored in metadata (which would cause team guardrails to be
|
||||
# silently skipped while default_on=True policy guardrails still fire).
|
||||
for meta_key in ("metadata", "litellm_metadata"):
|
||||
meta = data.get(meta_key) or {}
|
||||
if isinstance(meta, dict) and "guardrails" in meta:
|
||||
return meta.get("guardrails") or []
|
||||
return []
|
||||
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
|
||||
return [
|
||||
*_guardrails_listed_in(request_fields.get("metadata")),
|
||||
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
|
||||
]
|
||||
|
||||
def _guardrail_is_in_requested_guardrails(
|
||||
self,
|
||||
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
|
||||
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
|
||||
) -> bool:
|
||||
for _guardrail in requested_guardrails:
|
||||
if isinstance(_guardrail, dict):
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
Tracer = _Tracer
|
||||
Context = _Context
|
||||
SpanExporter = _SpanExporter
|
||||
|
|
@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span = None
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_span is guaranteed to be not None here
|
||||
if hasattr(parent_span, "set_status"):
|
||||
if parent_span is not None and hasattr(parent_span, "set_status"):
|
||||
parent_span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
# Raw-request as direct child of parent_span
|
||||
|
|
@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# record error on parent span (keeps hierarchy shallow)
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_otel_span is guaranteed to be not None here
|
||||
if parent_otel_span.is_recording():
|
||||
if parent_otel_span is not None and parent_otel_span.is_recording():
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
|
||||
|
|
|
|||
|
|
@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
(t for t in tools if is_anthropic_native_web_search_tool(t)),
|
||||
None,
|
||||
)
|
||||
emit_native_blocks: Final = native_tool is not None or (
|
||||
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
|
||||
)
|
||||
|
||||
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
|
||||
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
|
||||
|
||||
content: Final[list[dict[str, object]]] = []
|
||||
if native_tool is not None:
|
||||
if emit_native_blocks:
|
||||
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
|
||||
tool_name: Final = native_tool.get("name") or "web_search"
|
||||
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
|
||||
content.append(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
|
|
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
}
|
||||
)
|
||||
content.append(
|
||||
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
|
||||
WebSearchTransformation.build_web_search_outcome_block(
|
||||
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
|
||||
)
|
||||
)
|
||||
# Keep the text block so non-native short-circuit callers (Claude Code,
|
||||
# github_copilot, etc.) see the same payload they always have.
|
||||
content.append({"type": "text", "text": search_result_text})
|
||||
|
||||
response: Final[dict[str, object]] = {
|
||||
|
|
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
|
||||
len(search_result_text),
|
||||
native_tool is not None,
|
||||
emit_native_blocks,
|
||||
)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -412,6 +412,7 @@ class WebSearchTransformation:
|
|||
def build_web_search_tool_result_block(
|
||||
tool_use_id: str,
|
||||
search_response: SearchResponse | None,
|
||||
with_snippets: bool = True,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Build an Anthropic-native ``web_search_tool_result`` content block.
|
||||
|
|
@ -445,6 +446,8 @@ class WebSearchTransformation:
|
|||
``litellm.asearch()``. If None or empty, the block is still
|
||||
emitted with an empty result list (signals "search ran, no
|
||||
results" rather than "search did not run").
|
||||
with_snippets: False when a text block in the same reply already
|
||||
carries the snippets.
|
||||
"""
|
||||
items: Final[list[dict[str, object]]] = []
|
||||
if search_response is not None:
|
||||
|
|
@ -460,7 +463,7 @@ class WebSearchTransformation:
|
|||
"title": title,
|
||||
"page_age": page_age,
|
||||
"encrypted_content": "",
|
||||
"snippet": getattr(r, "snippet", "") or "",
|
||||
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
|
||||
}
|
||||
)
|
||||
return {
|
||||
|
|
@ -481,12 +484,15 @@ class WebSearchTransformation:
|
|||
}
|
||||
|
||||
@staticmethod
|
||||
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
|
||||
def build_web_search_outcome_block(
|
||||
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
|
||||
) -> dict[str, object]:
|
||||
match outcome:
|
||||
case SearchSucceeded(response=response):
|
||||
return WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id=tool_use_id,
|
||||
search_response=response,
|
||||
with_snippets=with_snippets,
|
||||
)
|
||||
case SearchFailed(error_code=error_code):
|
||||
return WebSearchTransformation.build_web_search_tool_result_error_block(
|
||||
|
|
|
|||
|
|
@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
|
|||
return getattr(chunk, "usage", None)
|
||||
|
||||
|
||||
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
|
||||
|
||||
|
||||
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
|
||||
|
||||
|
||||
def _numeric(value: object) -> float | None:
|
||||
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
|
||||
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
|
||||
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
|
||||
reading litellm's bill back as if the upstream had reported it."""
|
||||
hidden: Final[object] = getattr(response, "_hidden_params", None)
|
||||
if isinstance(hidden, dict):
|
||||
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
|
||||
upstream: Final = (
|
||||
hidden_params[UPSTREAM_USAGE_COST]
|
||||
if UPSTREAM_USAGE_COST in hidden_params
|
||||
else _numeric(getattr(usage, "cost", None))
|
||||
)
|
||||
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
|
||||
setattr(usage, "cost", bill)
|
||||
|
||||
|
||||
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
|
||||
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
|
||||
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
|
||||
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
|
||||
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
|
||||
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
|
||||
return None
|
||||
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
|
||||
cost: Final = (
|
||||
_numeric(hidden[UPSTREAM_USAGE_COST])
|
||||
if hidden is not None and UPSTREAM_USAGE_COST in hidden
|
||||
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
)
|
||||
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
|
|||
Values outside the range fall back to the default and warn, so a misconfigured knob can
|
||||
neither crash the caller nor silently change the meaning of what it computes.
|
||||
"""
|
||||
value: Final = get_env_int(env_var, default)
|
||||
return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
|
||||
|
||||
|
||||
def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
|
||||
"""Float counterpart of `get_env_int_in_range`."""
|
||||
return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
|
||||
|
||||
|
||||
def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
|
||||
if minimum <= value <= maximum:
|
||||
return value
|
||||
logging.getLogger("LiteLLM").warning(
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from ..exceptions import (
|
|||
ServiceUnavailableError,
|
||||
Timeout,
|
||||
UnprocessableEntityError,
|
||||
error_response_without_credentials,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -305,6 +306,16 @@ def _litellm_proxy_response(
|
|||
)
|
||||
|
||||
|
||||
def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
|
||||
if not isinstance(response, httpx.Response):
|
||||
return response
|
||||
return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
|
||||
|
||||
|
||||
def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
|
||||
return response if response is not None and (response.headers or response.content) else None
|
||||
|
||||
|
||||
def _map_openai_exception(
|
||||
*,
|
||||
model: str,
|
||||
|
|
@ -315,7 +326,7 @@ def _map_openai_exception(
|
|||
exception_provider: str,
|
||||
extra_information: str,
|
||||
) -> None:
|
||||
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
|
||||
response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
|
||||
if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
|
||||
raise ContentPolicyViolationError(
|
||||
message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
|
||||
|
|
@ -378,6 +389,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif (
|
||||
("invalid_request_error" in error_str and "content_policy_violation" in error_str)
|
||||
|
|
@ -449,6 +461,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
body=getattr(original_exception, "body", None),
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif "Request too large" in error_str:
|
||||
raise RateLimitError(
|
||||
|
|
@ -479,6 +492,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
request=_request,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif hasattr(original_exception, "status_code"):
|
||||
if original_exception.status_code == 400:
|
||||
|
|
@ -512,6 +526,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
raise BadRequestError(
|
||||
|
|
@ -563,6 +578,7 @@ def _map_openai_exception(
|
|||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
exception_status_code=original_exception.status_code,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
raise APIError(
|
||||
|
|
@ -572,6 +588,7 @@ def _map_openai_exception(
|
|||
model=model,
|
||||
request=getattr(original_exception, "request", None),
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
|
|
@ -2019,6 +2036,7 @@ def _map_azure_exception(
|
|||
exception_provider: str,
|
||||
extra_information: str,
|
||||
) -> None:
|
||||
response: Final = _without_credentials(getattr(original_exception, "response", None))
|
||||
message = get_error_message(error_obj=original_exception)
|
||||
if message is None:
|
||||
if hasattr(original_exception, "message"):
|
||||
|
|
@ -2054,7 +2072,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif "DeploymentNotFound" in error_str:
|
||||
raise NotFoundError(
|
||||
|
|
@ -2062,7 +2080,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
|
||||
error_str
|
||||
|
|
@ -2076,6 +2094,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
extra_information=extra_information,
|
||||
original_exception=original_exception,
|
||||
response=response,
|
||||
)
|
||||
elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
|
||||
helpful_message: Final = (
|
||||
|
|
@ -2094,7 +2113,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif getattr(original_exception, "status_code", None) in (None, 400) and (
|
||||
|
|
@ -2106,7 +2125,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
|
||||
|
|
@ -2115,7 +2134,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
|
||||
|
|
@ -2124,7 +2143,7 @@ def _map_azure_exception(
|
|||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif "Connection error" in error_str:
|
||||
raise APIConnectionError(
|
||||
|
|
@ -2140,7 +2159,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif original_exception.status_code == 401:
|
||||
|
|
@ -2149,7 +2168,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 404:
|
||||
raise NotFoundError(
|
||||
|
|
@ -2157,7 +2176,7 @@ def _map_azure_exception(
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 408:
|
||||
raise Timeout(
|
||||
|
|
@ -2165,6 +2184,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
llm_provider="azure",
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
raise BadRequestError(
|
||||
|
|
@ -2172,7 +2192,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
raise RateLimitError(
|
||||
|
|
@ -2180,7 +2200,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 502:
|
||||
raise BadGatewayError(
|
||||
|
|
@ -2188,7 +2208,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 503:
|
||||
raise ServiceUnavailableError(
|
||||
|
|
@ -2196,7 +2216,7 @@ def _map_azure_exception(
|
|||
model=model,
|
||||
llm_provider="azure",
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
)
|
||||
elif original_exception.status_code == 504: # gateway timeout error
|
||||
raise Timeout(
|
||||
|
|
@ -2205,6 +2225,7 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
llm_provider="azure",
|
||||
exception_status_code=original_exception.status_code,
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
raise APIError(
|
||||
|
|
@ -2214,6 +2235,7 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://openai.com/"),
|
||||
response=_upstream_reply(response),
|
||||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
|
|
@ -2589,17 +2611,20 @@ def exception_type(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=_upstream_reply(
|
||||
_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
custom_llm_provider == "litellm_proxy"
|
||||
): # handle special case where calling litellm proxy + exception str contains error message
|
||||
extract_and_raise_litellm_exception(
|
||||
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
|
||||
response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
|
||||
error_str=error_str,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
body=getattr(original_exception, "body", None),
|
||||
body=getattr(mappable_exception, "body", None),
|
||||
)
|
||||
if (
|
||||
custom_llm_provider == "openai"
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -292,6 +292,10 @@ def get_llm_provider(
|
|||
elif endpoint == "api.deepinfra.com/v1/openai":
|
||||
custom_llm_provider = "deepinfra"
|
||||
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
|
||||
elif endpoint == "api.clfaigateway.dev/v1":
|
||||
custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch
|
||||
clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
|
||||
dynamic_api_key = api_key or clf_env_key
|
||||
elif endpoint == "api.mistral.ai/v1":
|
||||
custom_llm_provider = "mistral"
|
||||
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
|
||||
|
|
@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
|
|||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "clf_ai_gateway":
|
||||
(
|
||||
api_base, # rebind-ok: resolved base/key are what this function returns, like every branch
|
||||
dynamic_api_key, # rebind-ok: see above
|
||||
) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "empower":
|
||||
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ Helper functions for health check calls.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
|
@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
|
|||
)
|
||||
|
||||
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
|
||||
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
|
||||
WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
|
||||
BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
|
||||
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
|
||||
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
|
||||
)
|
||||
|
|
@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
|
|||
return native(model, custom_llm_provider)
|
||||
|
||||
|
||||
def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
try:
|
||||
model_info: Final = ProviderConfigManager.get_provider_model_info(
|
||||
model=None, provider=LlmProviders(custom_llm_provider)
|
||||
)
|
||||
return None if model_info is None else model_info.get_base_model(model)
|
||||
except (ValueError, NotImplementedError):
|
||||
return None
|
||||
|
||||
|
||||
def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
|
||||
if custom_llm_provider == "bedrock":
|
||||
from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
|
||||
|
||||
return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
|
||||
base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
|
||||
return base_model is not None and not base_model.startswith(f"{segment}/")
|
||||
|
||||
|
||||
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
|
||||
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
provider_prefix: Final = f"{custom_llm_provider}/"
|
||||
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
|
||||
route_segments: Final = tuple(
|
||||
itertools.takewhile(
|
||||
lambda segment_index: _is_route_segment(
|
||||
custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
|
||||
),
|
||||
range(len(segments) - 1),
|
||||
)
|
||||
)
|
||||
route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
|
||||
model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
|
||||
ranked_model_ids: Final = tuple(
|
||||
(ranked_model.model.removeprefix(provider_prefix), ranked_model)
|
||||
for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
|
||||
)
|
||||
served_model_ids: Final = sorted(
|
||||
(ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
|
||||
key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
|
||||
)
|
||||
served_probes: Final = itertools.islice(
|
||||
dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
|
||||
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
|
||||
)
|
||||
provider_probes: Final = (
|
||||
f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
|
||||
)
|
||||
return tuple(dict.fromkeys((*served_probes, *provider_probes)))
|
||||
|
||||
|
||||
class HealthCheckHelpers:
|
||||
@staticmethod
|
||||
async def ahealth_check_wildcard_models(
|
||||
|
|
@ -99,23 +160,16 @@ class HealthCheckHelpers:
|
|||
litellm_logging_obj: "Logging",
|
||||
) -> dict:
|
||||
from litellm import acompletion
|
||||
from litellm.litellm_core_utils.llm_request_utils import (
|
||||
pick_cheapest_chat_models_from_llm_provider,
|
||||
)
|
||||
|
||||
# this is a wildcard model, we need to pick a random model from the provider
|
||||
cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
|
||||
if len(cheapest_models) == 0:
|
||||
probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
|
||||
if len(probe_models) == 0:
|
||||
raise Exception(
|
||||
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
||||
f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
|
||||
"no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
|
||||
)
|
||||
if len(cheapest_models) > 1:
|
||||
fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list
|
||||
else:
|
||||
fallback_models = None
|
||||
model_params["model"] = cheapest_models[0]
|
||||
model_params["model"] = probe_models[0]
|
||||
model_params["litellm_logging_obj"] = litellm_logging_obj
|
||||
model_params["fallbacks"] = fallback_models
|
||||
model_params["fallbacks"] = list(probe_models[1:]) or None
|
||||
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
|
||||
await acompletion(**model_params)
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
batch_chat_usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose
|
||||
|
|
@ -2091,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
if isinstance(priced_result, LiteLLMBatch) and (
|
||||
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
|
||||
):
|
||||
return 0.0
|
||||
|
||||
# Fallback: extract router_model_id from litellm_params when not available
|
||||
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
|
||||
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
|
||||
|
|
@ -3346,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
finally:
|
||||
self._restore_correlation_context()
|
||||
|
||||
def _is_batch_poll_without_logging(self, result: object) -> bool:
|
||||
litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
|
||||
return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
|
||||
def _success_handler_body(
|
||||
self,
|
||||
result: object = None,
|
||||
|
|
@ -3860,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## CALCULATE COST FOR BATCH JOBS
|
||||
if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
|
||||
litellm_params = self.litellm_params or {}
|
||||
litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
|
||||
if (
|
||||
litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
): # polling job will query these frequently, don't spam db logs
|
||||
if self._is_batch_poll_without_logging(result):
|
||||
return
|
||||
|
||||
batch_cost: Final = kwargs.get("batch_cost", None)
|
||||
|
|
@ -7481,7 +7487,8 @@ def get_standard_logging_object_payload(
|
|||
# Extract usage as a plain dict, avoiding Pydantic round-trip
|
||||
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
|
||||
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
|
||||
)
|
||||
usage_dict: Final = (
|
||||
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import datetime
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
|
|
@ -151,21 +156,54 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
Returns:
|
||||
list[str]: A list of the n cheapest chat models.
|
||||
"""
|
||||
return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RankedChatModel:
|
||||
model: str
|
||||
route_rank: int
|
||||
cost: float
|
||||
is_past_deprecation_date: bool
|
||||
|
||||
|
||||
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
|
||||
from litellm.utils import bundled_provider_models
|
||||
|
||||
if custom_llm_provider not in litellm.models_by_provider:
|
||||
return []
|
||||
|
||||
known_models: Final = bundled_provider_models(custom_llm_provider)
|
||||
ranked_rows: Final = (
|
||||
(route_rank, cost, model)
|
||||
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
|
||||
ranked_models: Final = (
|
||||
RankedChatModel(
|
||||
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
|
||||
)
|
||||
for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider)
|
||||
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
|
||||
)
|
||||
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
|
||||
return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost)))
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
|
||||
class _CostMapDeprecation(TypedDict):
|
||||
deprecation_date: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
|
||||
|
||||
|
||||
def _is_past_deprecation_date(cost_map_key: str) -> bool:
|
||||
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
).get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return (
|
||||
datetime.date.fromisoformat(deprecation_date)
|
||||
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
for model in models:
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
|
@ -173,7 +211,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
|
|||
continue
|
||||
if model_info.get("mode") != "chat":
|
||||
continue
|
||||
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
yield (
|
||||
model,
|
||||
(model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0),
|
||||
_is_past_deprecation_date(model_info["key"]),
|
||||
)
|
||||
|
||||
|
||||
def _route_rank(model: str, custom_llm_provider: str) -> int | None:
|
||||
|
|
|
|||
|
|
@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
|
|||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if current_content_type and current_content_type not in [
|
||||
if media_type and media_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return current_content_type
|
||||
return media_type
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
|
|
|
|||
|
|
@ -912,9 +912,9 @@ def create_anthropic_image_param(
|
|||
image_url_input: str | dict,
|
||||
format: str | None = None,
|
||||
is_bedrock_invoke: bool = False,
|
||||
) -> AnthropicMessagesImageParam:
|
||||
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
|
||||
"""
|
||||
Create an AnthropicMessagesImageParam from an image URL input.
|
||||
Create an Anthropic content block from an image URL input.
|
||||
|
||||
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
|
||||
"""
|
||||
|
|
@ -933,14 +933,7 @@ def create_anthropic_image_param(
|
|||
if is_bedrock_invoke or image_url.startswith("http://"):
|
||||
base64_url: Final = convert_url_to_base64(url=image_url)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
else:
|
||||
# HTTPS URL - pass directly for regular Anthropic
|
||||
return AnthropicMessagesImageParam(
|
||||
|
|
@ -953,14 +946,7 @@ def create_anthropic_image_param(
|
|||
else:
|
||||
# Convert to base64 for data URIs or other formats
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
|
||||
|
||||
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
|
||||
|
|
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
|
|||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
|
||||
anthropic_content_list.append(
|
||||
cast(
|
||||
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
|
||||
_anthropic_image_param,
|
||||
)
|
||||
)
|
||||
elif content["type"] == "tool_reference":
|
||||
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
|
||||
elif content["type"] == "file":
|
||||
|
|
@ -5878,38 +5869,36 @@ def function_call_prompt(
|
|||
]
|
||||
|
||||
|
||||
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
|
||||
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
|
||||
"""
|
||||
Decides if a user-defined custom prompt or default needs to be used
|
||||
|
||||
Returns the prompt str that's passed to the model as a user message
|
||||
"""
|
||||
custom_prompt_details: dict | None = None
|
||||
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
|
||||
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict[
|
||||
f"{model}/response_schema_prompt"
|
||||
] # allow user to define custom response schema prompt by model
|
||||
elif "response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
|
||||
|
||||
if custom_prompt_details is not None:
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
|
||||
)
|
||||
else:
|
||||
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
|
||||
model_prompt_key: Final = f"{model}/response_schema_prompt"
|
||||
custom_prompt_details: Final = prompts.get(
|
||||
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
|
||||
)
|
||||
if custom_prompt_details is None:
|
||||
return default_response_schema_prompt(response_schema=response_schema)
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details,
|
||||
messages=[{"role": "user", "content": f"{response_schema}"}],
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def default_response_schema_prompt(response_schema: dict) -> str:
|
||||
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Used if provider/model doesn't support 'response_schema' param.
|
||||
|
||||
This is the default prompt. Allow user to override this with a custom_prompt.
|
||||
"""
|
||||
prompt_str: Final = f"""Use this JSON schema:
|
||||
```json
|
||||
{response_schema}
|
||||
```json
|
||||
{json.dumps(response_schema, default=str)}
|
||||
```"""
|
||||
return prompt_str
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
|
||||
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
|
||||
from litellm.llms.base_llm.realtime.transformation import (
|
||||
BaseRealtimeConfig,
|
||||
RealtimeBackend,
|
||||
RealtimeSessionUpdateRejection,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
OpenAIRealtimeEvents,
|
||||
OpenAIRealtimeOutputItemDone,
|
||||
|
|
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_MERGE_MAX_DEPTH: Final = 8
|
||||
|
||||
|
||||
class _SessionUpdateFrame(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
session: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
def _merge_session_fields(
|
||||
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
|
||||
) -> Mapping[str, object]:
|
||||
def merged_value(key: str) -> object:
|
||||
if key not in update:
|
||||
return base[key]
|
||||
value: Final = update[key]
|
||||
base_value: Final = base.get(key)
|
||||
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
|
||||
return _merge_session_fields(
|
||||
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
|
||||
)
|
||||
return update[key]
|
||||
|
||||
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
|
||||
|
||||
|
||||
def _session_update_frame(session: Mapping[str, object]) -> str:
|
||||
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
|
||||
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
|
||||
|
||||
|
||||
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
|
||||
code="session_update_too_large",
|
||||
message=(
|
||||
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
|
||||
"so this session.update was not applied"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
|
||||
|
||||
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "audio_input_not_screenable",
|
||||
"message": (
|
||||
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
|
||||
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
|
||||
),
|
||||
"param": None,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _decode_json_object(payload: str) -> Mapping[str, object]:
|
||||
"""Decode a realtime frame into its top-level field mapping."""
|
||||
return json.loads(payload)
|
||||
|
|
@ -243,8 +309,17 @@ class RealTimeStreaming:
|
|||
event_normalizer: RealtimeEventNormalizer | None = None,
|
||||
usage_meter: RealtimeUsageMeter | None = None,
|
||||
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
|
||||
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.websocket: _ClientWebSocket = websocket
|
||||
self._clock: Final = clock
|
||||
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
|
||||
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
|
||||
self._held_client_content: tuple[str, ...] = ()
|
||||
self._held_client_content_bytes: int = 0
|
||||
self._held_client_content_deadline: float | None = None
|
||||
self._replay_scope: Final = client_websocket_scope(websocket)
|
||||
self.backend_ws = backend_ws
|
||||
self.logging_obj = logging_obj
|
||||
|
|
@ -300,6 +375,10 @@ class RealTimeStreaming:
|
|||
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
|
||||
# content (realtimeInput / clientContent / toolResponse) has been sent.
|
||||
self._content_sent_after_setup: bool = False
|
||||
self._held_session_update: Mapping[str, object] | None = None
|
||||
self._applied_session_fields: Mapping[str, object] | None = None
|
||||
self._held_session_update_bytes: int = 0
|
||||
self._unscreened_audio_refused: bool = False
|
||||
# Whether this is a transcription-only session (session.type == "transcription",
|
||||
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
|
||||
# their input_audio_transcription.completed usage drives duration-based cost.
|
||||
|
|
@ -337,6 +416,17 @@ class RealTimeStreaming:
|
|||
]
|
||||
)
|
||||
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
|
||||
_CLIENT_CONTENT_TYPES = frozenset(
|
||||
[
|
||||
"conversation.item.create",
|
||||
"response.create",
|
||||
"input_audio_buffer.append",
|
||||
"input_audio_buffer.commit",
|
||||
"input_audio_buffer.end",
|
||||
]
|
||||
)
|
||||
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
|
||||
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
|
||||
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
|
||||
"pcm16": {"type": "audio/pcm", "rate": 24000},
|
||||
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
|
||||
|
|
@ -377,7 +467,7 @@ class RealTimeStreaming:
|
|||
message_obj: dict[str, Any] = cast(dict[str, Any], message)
|
||||
else:
|
||||
message_obj = cast(dict[str, Any], json.loads(message))
|
||||
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
|
||||
self._collect_tool_calls_from_response_done(message_obj)
|
||||
if not self._should_store_message(message_obj):
|
||||
return
|
||||
try:
|
||||
|
|
@ -502,7 +592,7 @@ class RealTimeStreaming:
|
|||
self.store_message(flush_event)
|
||||
self._capture_transcription_usage(flush_event)
|
||||
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
|
||||
"""Extract function_call items from response.done events for spend logging."""
|
||||
try:
|
||||
if event_obj.get("type") != "response.done":
|
||||
|
|
@ -607,6 +697,167 @@ class RealTimeStreaming:
|
|||
self._usage_meter.observe_client_event(sent_event)
|
||||
return True
|
||||
|
||||
def _builds_setup_lazily(self) -> bool:
|
||||
if self.provider_config is None:
|
||||
return False
|
||||
return self.provider_config.builds_setup_from_session_update()
|
||||
|
||||
def _holds_session_updates_until_first_content(self) -> bool:
|
||||
return self.session_configuration_request is None and self._builds_setup_lazily()
|
||||
|
||||
async def _rejected_setup_session_update(self, message: str) -> bool:
|
||||
provider_config: Final = self.provider_config
|
||||
if provider_config is None or not provider_config.builds_setup_from_session_update():
|
||||
return False
|
||||
client_event: Final = _decode_json_object(message)
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
if self._leaves_applied_setup_unchanged(update):
|
||||
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
|
||||
return True
|
||||
rejection: Final = provider_config.setup_session_update_rejection(
|
||||
self.model, update, setup_sent=self.session_configuration_request is not None
|
||||
)
|
||||
if rejection is None:
|
||||
return False
|
||||
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
|
||||
return True
|
||||
|
||||
def _held_session_update_message(self) -> str:
|
||||
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
|
||||
|
||||
async def _hold_session_update(self, message: str) -> None:
|
||||
client_event: Final = _decode_json_object(message)
|
||||
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
|
||||
)
|
||||
return
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
|
||||
self._held_session_update = held
|
||||
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
|
||||
await self._acknowledge_session_setup(held)
|
||||
|
||||
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
|
||||
if self.provider_config is None:
|
||||
return None
|
||||
preview: Final = self.provider_config.transform_realtime_request(
|
||||
_session_update_frame(session), self.model, None
|
||||
)
|
||||
return next(
|
||||
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
|
||||
)
|
||||
|
||||
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
|
||||
applied: Final = self._applied_session_fields
|
||||
if self.session_configuration_request is None or applied is None:
|
||||
return False
|
||||
applied_setup: Final = self._setup_preview(applied)
|
||||
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
|
||||
return (
|
||||
applied_setup is not None
|
||||
and merged_setup is not None
|
||||
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
|
||||
)
|
||||
|
||||
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
|
||||
if self.provider_config is None:
|
||||
return
|
||||
setup: Final = self._setup_preview(session)
|
||||
if setup is None:
|
||||
return
|
||||
acknowledged: Final = self.provider_config.transform_session_created_event(
|
||||
self.model, str(self.logging_obj.litellm_trace_id), setup
|
||||
)
|
||||
if acknowledged is None:
|
||||
return
|
||||
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
|
||||
event_str: Final = json.dumps(event)
|
||||
self.store_message(event_str)
|
||||
await self._send_event_to_client(event, event_str)
|
||||
|
||||
async def _send_held_session_update_as_setup(self) -> None:
|
||||
held: Final = self._held_session_update
|
||||
if held is None:
|
||||
return
|
||||
message: Final = self._held_session_update_message()
|
||||
self._held_session_update = None
|
||||
self._held_session_update_bytes = 0
|
||||
if await self._send_to_backend(message):
|
||||
self._applied_session_fields = held
|
||||
await self._send_held_client_content()
|
||||
|
||||
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
|
||||
held_types: Final = (
|
||||
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
|
||||
if self._uses_deferred_backend_setup()
|
||||
else RealTimeStreaming._HELD_CONTENT_TYPES
|
||||
)
|
||||
return (
|
||||
msg_type in held_types
|
||||
and self._held_session_update is not None
|
||||
and self._holds_session_updates_until_first_content()
|
||||
and not self._pending_messages_until_setup
|
||||
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
|
||||
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
|
||||
)
|
||||
|
||||
def _hold_client_content(self, message: str) -> None:
|
||||
if self._held_client_content_deadline is None:
|
||||
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
|
||||
self._held_client_content = (*self._held_client_content, message)
|
||||
self._held_client_content_bytes += len(message.encode("utf-8"))
|
||||
|
||||
async def _send_held_client_content(self) -> None:
|
||||
held: Final = self._held_client_content
|
||||
self._held_client_content = ()
|
||||
self._held_client_content_bytes = 0
|
||||
self._held_client_content_deadline = None
|
||||
for message in held:
|
||||
await self._send_to_backend(message)
|
||||
|
||||
def _setup_deadline(self) -> float:
|
||||
content_deadline: Final = self._held_client_content_deadline
|
||||
return (
|
||||
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
|
||||
)
|
||||
|
||||
async def _send_setup_before_content(self, msg_type: str | None) -> None:
|
||||
if self._holds_session_updates_until_first_content():
|
||||
await self._send_held_session_update_as_setup()
|
||||
await self._send_default_setup_before_first_client_message(msg_type)
|
||||
|
||||
async def _receive_client_message_before_setup_deadline(self) -> str | None:
|
||||
if not self._holds_session_updates_until_first_content():
|
||||
return await self._receive_client_message()
|
||||
receive: Final = asyncio.ensure_future(self._receive_client_message())
|
||||
try:
|
||||
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
|
||||
if not done:
|
||||
await self._send_setup_before_content(None)
|
||||
return await receive
|
||||
finally:
|
||||
receive.cancel()
|
||||
|
||||
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
|
||||
if self.session_configuration_request is not None or self.provider_config is None:
|
||||
return
|
||||
if msg_type == "session.update" or (
|
||||
self._uses_deferred_backend_setup()
|
||||
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
|
||||
):
|
||||
return
|
||||
default_setup: Final = self.provider_config.session_configuration_request(self.model)
|
||||
if not default_setup:
|
||||
return
|
||||
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
|
||||
await self.backend_ws.send(setup)
|
||||
self._cache_session_configuration_request(setup)
|
||||
self._applied_session_fields = _NO_SESSION_FIELDS
|
||||
await self._send_held_client_content()
|
||||
|
||||
def _enforce_transcription_session_model(self, message: str) -> str:
|
||||
"""Force client transcription session updates to the authorized model.
|
||||
|
||||
|
|
@ -851,6 +1102,20 @@ class RealTimeStreaming:
|
|||
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
|
||||
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
|
||||
|
||||
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
|
||||
if pre_block_backend_message is None and self._is_transcription_session:
|
||||
return
|
||||
await self._send_setup_before_content("conversation.item.create")
|
||||
# Deliver any caller-supplied backend message FIRST so that
|
||||
# protocol contracts requiring a specific ordering (e.g.
|
||||
# Gemini Live's mandatory ``toolResponse`` after a
|
||||
# ``toolCall``) are honored before the guardrail's own
|
||||
# clientContent / cancel messages are sent.
|
||||
if pre_block_backend_message is not None:
|
||||
await self._send_to_backend(pre_block_backend_message)
|
||||
if not self._is_transcription_session:
|
||||
await self._withdraw_blocked_turn(item_id)
|
||||
|
||||
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
|
||||
await self._send_guardrail_cancel()
|
||||
if item_id is not None and self._deletes_conversation_items:
|
||||
|
|
@ -958,6 +1223,21 @@ class RealTimeStreaming:
|
|||
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
|
||||
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
|
||||
|
||||
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
|
||||
return (
|
||||
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
|
||||
and self._builds_setup_lazily()
|
||||
and self._should_disable_vad_auto_response()
|
||||
)
|
||||
|
||||
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
|
||||
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
|
||||
return
|
||||
self._unscreened_audio_refused = True
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
|
||||
)
|
||||
|
||||
async def run_realtime_guardrails(
|
||||
self,
|
||||
transcript: str,
|
||||
|
|
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
|
|||
# Use realtime_violation_message if configured; fall back to guardrail error text.
|
||||
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
|
||||
|
||||
# Deliver any caller-supplied backend message FIRST so that
|
||||
# protocol contracts requiring a specific ordering (e.g.
|
||||
# Gemini Live's mandatory ``toolResponse`` after a
|
||||
# ``toolCall``) are honored before the guardrail's own
|
||||
# clientContent / cancel messages are sent.
|
||||
if pre_block_backend_message is not None:
|
||||
await self._send_to_backend(pre_block_backend_message)
|
||||
if not self._is_transcription_session:
|
||||
await self._withdraw_blocked_turn(item_id)
|
||||
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
|
||||
# Send the policy violation hint (shows as small gray status text in UI).
|
||||
await self.websocket.send_text(
|
||||
json.dumps(
|
||||
|
|
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
|
|||
break
|
||||
finally:
|
||||
self._flushing_pending_messages_until_setup = False
|
||||
if self.session_created_sent_to_client and self._builds_setup_lazily():
|
||||
await self._maybe_send_guardrail_turn_detection_update()
|
||||
continue
|
||||
if self.session_created_sent_to_client:
|
||||
# A synthetic session.created (with placeholder defaults) was
|
||||
# already forwarded to the client when we connected. The
|
||||
|
|
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
|
|||
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
|
||||
|
||||
def _backend_refused_session(self, close: BackendClose) -> bool:
|
||||
return close.code != 1000 and not self._backend_sent_frames
|
||||
return (
|
||||
close.code != 1000
|
||||
and not self._backend_sent_frames
|
||||
and not self._holds_session_updates_until_first_content()
|
||||
)
|
||||
|
||||
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
|
||||
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
|
||||
|
|
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
|
|||
client_event: _ClientEventFrame
|
||||
try:
|
||||
while True:
|
||||
message = await self._receive_client_message()
|
||||
message = await self._receive_client_message_before_setup_deadline()
|
||||
if message is None:
|
||||
return ClientLoopExit.CLIENT_DISCONNECTED
|
||||
|
||||
|
|
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
|
|||
if (
|
||||
msg_type == "session.update"
|
||||
and self.session_configuration_request is None
|
||||
and self._held_session_update is None
|
||||
and not self._guardrail_turn_detection_update_sent
|
||||
and self._should_disable_vad_auto_response()
|
||||
and (
|
||||
|
|
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
|
|||
except (json.JSONDecodeError, AttributeError):
|
||||
pass
|
||||
|
||||
if self._refuses_unscreened_audio(msg_type):
|
||||
await self._refuse_unscreened_audio(msg_type, message)
|
||||
continue
|
||||
|
||||
## LOGGING
|
||||
# Log after any in-place modifications (GA remap, guardrail
|
||||
# turn_detection injection) so audit logs reflect what we
|
||||
# actually forward to the backend.
|
||||
self.store_input(message=message)
|
||||
|
||||
if self._holds_client_content(msg_type, message):
|
||||
self._hold_client_content(message)
|
||||
continue
|
||||
if self._held_client_content and msg_type != "session.update":
|
||||
await self._send_setup_before_content(msg_type)
|
||||
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
|
||||
await self._send_setup_before_content(msg_type)
|
||||
|
||||
if self._should_buffer_client_message_until_setup(message):
|
||||
self._buffer_pending_message_until_setup(message)
|
||||
continue
|
||||
|
|
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
|
|||
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
|
||||
if msg_type == "session.update" and self.provider_config is None:
|
||||
await self._send_session_gate_once()
|
||||
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
|
||||
continue
|
||||
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
|
||||
await self._hold_session_update(message)
|
||||
if self._pending_messages_until_setup:
|
||||
await self._send_held_session_update_as_setup()
|
||||
continue
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
|
||||
await self._send_setup_before_content(msg_type)
|
||||
try:
|
||||
sent = await self._send_to_backend(message)
|
||||
except RealtimeTranscriptionProtocolError as e:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import builtins
|
|||
import time
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import chain, groupby
|
||||
from itertools import chain, groupby, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
|
||||
|
||||
|
|
@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
USAGE_COST_BILLED_PROVIDERS,
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
chunk_usage,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
|
|
@ -257,7 +258,7 @@ def apply_grounding_request_counts(
|
|||
|
||||
|
||||
class ChunkProcessor:
|
||||
def __init__(self, chunks: list, messages: Sequence | None = None):
|
||||
def __init__(self, chunks: list, messages: Sequence[object] | None = None):
|
||||
self.chunks = self._sort_chunks(chunks)
|
||||
self.messages = messages
|
||||
self.first_chunk = chunks[0]
|
||||
|
|
@ -390,6 +391,30 @@ class ChunkProcessor:
|
|||
case _:
|
||||
return "assistant"
|
||||
|
||||
@staticmethod
|
||||
def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object:
|
||||
finish_reason = unreported
|
||||
for chunk in chunks:
|
||||
if "choices" in chunk and len(chunk["choices"]) > 0:
|
||||
chunk_finish_reason = None
|
||||
if hasattr(chunk["choices"][0], "finish_reason"):
|
||||
chunk_finish_reason = chunk["choices"][0].finish_reason
|
||||
elif "finish_reason" in chunk["choices"][0]:
|
||||
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
|
||||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
return finish_reason
|
||||
|
||||
@staticmethod
|
||||
def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices:
|
||||
return Choices(
|
||||
index=index,
|
||||
finish_reason=ChunkProcessor._finish_reason_of_chunks(
|
||||
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
|
||||
),
|
||||
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
|
||||
"""
|
||||
|
|
@ -414,7 +439,11 @@ class ChunkProcessor:
|
|||
None,
|
||||
)
|
||||
|
||||
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
|
||||
def build_base_response(
|
||||
self,
|
||||
chunks: Sequence["_BaseChunk"],
|
||||
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
|
||||
) -> ModelResponse:
|
||||
chunk = self.first_chunk
|
||||
id: Final = ChunkProcessor._get_chunk_id(chunks)
|
||||
object: Final = chunk["object"]
|
||||
|
|
@ -426,16 +455,7 @@ class ChunkProcessor:
|
|||
service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks)
|
||||
|
||||
role: Final = ChunkProcessor._get_role_from_chunks(chunks)
|
||||
finish_reason = "stop"
|
||||
for chunk in chunks:
|
||||
if "choices" in chunk and len(chunk["choices"]) > 0:
|
||||
chunk_finish_reason = None
|
||||
if hasattr(chunk["choices"][0], "finish_reason"):
|
||||
chunk_finish_reason = chunk["choices"][0].finish_reason
|
||||
elif "finish_reason" in chunk["choices"][0]:
|
||||
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
|
||||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
|
||||
|
||||
# Initialize the response dictionary
|
||||
response = ModelResponse(
|
||||
|
|
@ -455,13 +475,17 @@ class ChunkProcessor:
|
|||
if service_tier is not None
|
||||
else MappingProxyType({})
|
||||
),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
"choices": (
|
||||
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
|
||||
if len(choice_streams) > 1
|
||||
else [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
]
|
||||
),
|
||||
"usage": {
|
||||
"prompt_tokens": 0, # Modify as needed
|
||||
"completion_tokens": 0, # Modify as needed
|
||||
|
|
@ -1357,9 +1381,6 @@ def stream_chunk_builder_text_completion(
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
|
||||
|
||||
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
if logging_obj is None:
|
||||
return None
|
||||
|
|
@ -1367,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
|||
return provider
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> float | None:
|
||||
|
|
@ -1381,9 +1398,6 @@ def _stream_builder_response_cost(
|
|||
for chunk in reversed(chunks)
|
||||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
|
||||
|
|
@ -1430,13 +1444,15 @@ def _set_stream_builder_response_cost(
|
|||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
|
||||
getattr(usage, "cost", None), (int, float)
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
|
||||
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
|
||||
return
|
||||
setattr(usage, "cost", computed_cost)
|
||||
_set_hidden_response_cost(response, float(computed_cost))
|
||||
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
|
||||
|
||||
|
||||
def _field_of(value: object, name: str) -> object:
|
||||
|
|
@ -1520,11 +1536,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
|
|||
return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
|
||||
|
||||
|
||||
def _stream_choice_index(choice: object) -> int:
|
||||
index: Final = _field_of(choice, "index")
|
||||
return index if isinstance(index, int) else 0
|
||||
|
||||
|
||||
def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool:
|
||||
return any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"]
|
||||
for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks)
|
||||
)
|
||||
|
||||
|
||||
def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]:
|
||||
indices: Final = sorted(
|
||||
frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks)))
|
||||
)
|
||||
if len(indices) <= 1:
|
||||
return ((0, chunks),)
|
||||
return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices)
|
||||
|
||||
|
||||
def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]:
|
||||
return tuple(
|
||||
_BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index])
|
||||
for chunk in chunks
|
||||
)
|
||||
|
||||
|
||||
def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None:
|
||||
choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices"))
|
||||
parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk)))
|
||||
return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None
|
||||
|
||||
|
||||
def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]:
|
||||
tool_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
|
||||
for choice in chunk.get("choices") or ()
|
||||
)
|
||||
]
|
||||
|
||||
if tool_call_chunks:
|
||||
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
|
||||
yield "content", None
|
||||
yield "tool_calls", tool_calls_list
|
||||
|
||||
function_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "function_call" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["function_call"] is not None
|
||||
]
|
||||
|
||||
if function_call_chunks:
|
||||
yield "content", None
|
||||
yield "function_call", processor.get_combined_function_call_content(function_call_chunks)
|
||||
|
||||
content_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["content"] is not None
|
||||
]
|
||||
|
||||
if content_chunks:
|
||||
yield "content", processor.get_combined_content(content_chunks)
|
||||
|
||||
thinking_blocks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "thinking_blocks" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
|
||||
]
|
||||
|
||||
if thinking_blocks:
|
||||
yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks)
|
||||
|
||||
reasoning_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "reasoning_content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
|
||||
]
|
||||
|
||||
if reasoning_chunks:
|
||||
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
yield "reasoning_items", list(reasoning_items)
|
||||
|
||||
annotation_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "annotations" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["annotations"] is not None
|
||||
]
|
||||
|
||||
message_annotations, provider_annotations = (
|
||||
partition_annotations(
|
||||
list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks))
|
||||
)
|
||||
if annotation_chunks
|
||||
else (None, ())
|
||||
)
|
||||
if message_annotations is not None:
|
||||
yield "annotations", message_annotations
|
||||
|
||||
audio_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "audio" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["audio"] is not None
|
||||
]
|
||||
|
||||
if audio_chunks:
|
||||
yield "audio", processor.get_combined_audio_content(audio_chunks)
|
||||
|
||||
# Handle image chunks from models like gemini-2.5-flash-image
|
||||
# See: https://github.com/BerriAI/litellm/issues/19478
|
||||
image_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "images" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["images"] is not None
|
||||
]
|
||||
|
||||
if image_chunks:
|
||||
# Images come complete in a single chunk, collect all images from all chunks
|
||||
all_images: Final = []
|
||||
for chunk in image_chunks:
|
||||
all_images.extend(chunk["choices"][0]["delta"]["images"])
|
||||
yield "images", all_images
|
||||
|
||||
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
|
||||
# See: https://github.com/BerriAI/litellm/issues/17737
|
||||
provider_specific_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "provider_specific_fields" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
|
||||
]
|
||||
|
||||
provider_field_dicts: Final = tuple(
|
||||
fields
|
||||
for chunk in provider_specific_chunks
|
||||
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
|
||||
if isinstance(fields, dict)
|
||||
)
|
||||
streamed_citations: Final = tuple(
|
||||
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
|
||||
)
|
||||
citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
|
||||
provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {}
|
||||
combined_provider_fields: Final = {
|
||||
key: value
|
||||
for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts)
|
||||
for key, value in fields.items()
|
||||
if key != "citation"
|
||||
}
|
||||
|
||||
if combined_provider_fields:
|
||||
yield "provider_specific_fields", combined_provider_fields
|
||||
|
||||
|
||||
def stream_chunk_builder(
|
||||
chunks: list,
|
||||
messages: Sequence | None = None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
start_time: object = None,
|
||||
end_time: object = None,
|
||||
logging_obj: Optional["Logging"] = None,
|
||||
count_prompt_tokens: Callable[[], int] | None = None,
|
||||
) -> ModelResponse | TextCompletionResponse | None:
|
||||
|
|
@ -1561,26 +1754,15 @@ def stream_chunk_builder(
|
|||
usage_model: Final = usage_tokenizer_model(
|
||||
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
|
||||
)
|
||||
# Initialize the response dictionary
|
||||
response: Final = processor.build_base_response(chunks)
|
||||
choice_streams: Final = _choice_streams(chunks)
|
||||
response: Final = processor.build_base_response(chunks, choice_streams)
|
||||
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
|
||||
|
||||
# Fast path for the common text-only streaming case:
|
||||
# avoid repeated multi-pass list scans over chunks.
|
||||
simple_content_parts: Final[list[str]] = []
|
||||
is_simple_text_stream = True
|
||||
for chunk in chunks:
|
||||
if not chunk.get("choices"):
|
||||
continue
|
||||
|
||||
if (part := _simple_text_part(chunk["choices"])) is None:
|
||||
is_simple_text_stream = False
|
||||
break
|
||||
if part:
|
||||
simple_content_parts.append(part)
|
||||
|
||||
if is_simple_text_stream:
|
||||
if simple_content_parts:
|
||||
response["choices"][0]["message"]["content"] = "".join(simple_content_parts)
|
||||
simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams)
|
||||
if None not in simple_texts:
|
||||
for (choice, _), simple_text in zip(rebuilt_choices, simple_texts):
|
||||
if simple_text:
|
||||
choice.message.content = simple_text
|
||||
completion_output = get_content_from_model_response(response)
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks,
|
||||
|
|
@ -1619,165 +1801,9 @@ def stream_chunk_builder(
|
|||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
return response
|
||||
|
||||
tool_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
|
||||
for choice in chunk.get("choices") or ()
|
||||
)
|
||||
]
|
||||
|
||||
if len(tool_call_chunks) > 0:
|
||||
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
|
||||
_choice = response.choices[0]
|
||||
_choice.message.content = None
|
||||
_choice.message.tool_calls = tool_calls_list
|
||||
|
||||
function_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "function_call" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["function_call"] is not None
|
||||
]
|
||||
|
||||
if len(function_call_chunks) > 0:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.content = None
|
||||
_choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks)
|
||||
|
||||
content_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["content"] is not None
|
||||
]
|
||||
|
||||
if len(content_chunks) > 0:
|
||||
response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks)
|
||||
|
||||
thinking_blocks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "thinking_blocks" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
|
||||
]
|
||||
|
||||
if len(thinking_blocks) > 0:
|
||||
response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content(
|
||||
thinking_blocks
|
||||
)
|
||||
|
||||
reasoning_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "reasoning_content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
|
||||
]
|
||||
|
||||
if len(reasoning_chunks) > 0:
|
||||
response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content(
|
||||
reasoning_chunks
|
||||
)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
|
||||
|
||||
annotation_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "annotations" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["annotations"] is not None
|
||||
]
|
||||
|
||||
if len(annotation_chunks) > 0:
|
||||
# Merge annotations from ALL chunks — providers may spread
|
||||
# them across multiple streaming chunks or send them only in
|
||||
# the final chunk.
|
||||
all_annotations: Final[list[object]] = []
|
||||
for ac in annotation_chunks:
|
||||
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
|
||||
message_annotations, provider_annotations = partition_annotations(all_annotations)
|
||||
if message_annotations is not None:
|
||||
response["choices"][0]["message"]["annotations"] = message_annotations
|
||||
if provider_annotations:
|
||||
response.choices[0].message.provider_specific_fields = {
|
||||
**(response.choices[0].message.provider_specific_fields or {}),
|
||||
"annotations": list(provider_annotations),
|
||||
}
|
||||
|
||||
audio_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "audio" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["audio"] is not None
|
||||
]
|
||||
|
||||
if len(audio_chunks) > 0:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.audio = processor.get_combined_audio_content(audio_chunks)
|
||||
|
||||
# Handle image chunks from models like gemini-2.5-flash-image
|
||||
# See: https://github.com/BerriAI/litellm/issues/19478
|
||||
image_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "images" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["images"] is not None
|
||||
]
|
||||
|
||||
if len(image_chunks) > 0:
|
||||
# Images come complete in a single chunk, collect all images from all chunks
|
||||
all_images: Final = []
|
||||
for chunk in image_chunks:
|
||||
all_images.extend(chunk["choices"][0]["delta"]["images"])
|
||||
response["choices"][0]["message"]["images"] = all_images
|
||||
|
||||
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
|
||||
# See: https://github.com/BerriAI/litellm/issues/17737
|
||||
provider_specific_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "provider_specific_fields" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
|
||||
]
|
||||
|
||||
if len(provider_specific_chunks) > 0:
|
||||
provider_field_dicts: Final = tuple(
|
||||
fields
|
||||
for chunk in provider_specific_chunks
|
||||
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
|
||||
if isinstance(fields, dict)
|
||||
)
|
||||
streamed_citations: Final = tuple(
|
||||
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
|
||||
)
|
||||
citation_fields: Final = (
|
||||
{"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
|
||||
)
|
||||
combined_provider_fields: Final = {
|
||||
key: value
|
||||
for fields in (citation_fields, *provider_field_dicts)
|
||||
for key, value in fields.items()
|
||||
if key != "citation"
|
||||
}
|
||||
|
||||
if combined_provider_fields:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.provider_specific_fields = {
|
||||
**(_choice.message.provider_specific_fields or {}),
|
||||
**combined_provider_fields,
|
||||
}
|
||||
for choice, choice_chunks in rebuilt_choices:
|
||||
for field, value in _assembled_message_fields(processor, choice_chunks):
|
||||
choice.message[field] = value
|
||||
|
||||
completion_output = get_content_from_model_response(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,14 +7,17 @@ import re
|
|||
import struct
|
||||
from bisect import bisect_right
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from itertools import accumulate, chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
|
||||
|
||||
import anyio
|
||||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from pydantic import TypeAdapter
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
|
|
@ -33,7 +36,12 @@ from litellm.constants import (
|
|||
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
|
||||
MAX_TILE_HEIGHT,
|
||||
MAX_TILE_WIDTH,
|
||||
OPENAI_CHAT_MODES,
|
||||
OPENAI_MODEL_NAME_MAX_DASHES,
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
|
||||
PDF_DATA_URL_PREFIX,
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH,
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
|
||||
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
|
||||
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
|
||||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
|
|
@ -62,6 +70,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesToolResultParam,
|
||||
AnthropicMessagesToolUseParam,
|
||||
AnthropicSearchResultParam,
|
||||
CompactionBlock,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -606,6 +615,35 @@ TokenCounterFunction = Callable[[str], int]
|
|||
Type for a function that counts tokens in a string.
|
||||
"""
|
||||
|
||||
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
|
||||
|
||||
|
||||
class _PdfBox(Protocol):
|
||||
@property
|
||||
def width(self) -> float: ...
|
||||
|
||||
@property
|
||||
def height(self) -> float: ...
|
||||
|
||||
|
||||
class _PdfContent(Protocol):
|
||||
def get_data(self) -> bytes: ...
|
||||
|
||||
|
||||
class _PdfPage(Protocol):
|
||||
@property
|
||||
def mediabox(self) -> _PdfBox: ...
|
||||
|
||||
def extract_text(self) -> str: ...
|
||||
|
||||
def get_contents(self) -> _PdfContent | None: ...
|
||||
|
||||
|
||||
class _PdfReader(Protocol):
|
||||
@property
|
||||
def pages(self) -> Sequence[_PdfPage]: ...
|
||||
|
||||
|
||||
EXTRAPOLATION_SAMPLES: Final = 16
|
||||
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
|
||||
T_Retval = TypeVar("T_Retval")
|
||||
|
|
@ -662,7 +700,7 @@ class _MessageCountParams:
|
|||
if uses_legacy_message_accounting(model):
|
||||
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
|
||||
self.tokens_per_name = -1 # if there's a name, the role is omitted
|
||||
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
|
||||
elif _is_openai_chat_model(actual_model):
|
||||
self.tokens_per_message = 3
|
||||
self.tokens_per_name = 1
|
||||
else:
|
||||
|
|
@ -671,6 +709,63 @@ class _MessageCountParams:
|
|||
self.tokens_per_name = 1
|
||||
self.count_function = _get_count_function(model, custom_tokenizer)
|
||||
self.tool_section = chat_template_tool_section(model)
|
||||
self._model = model
|
||||
|
||||
@cached_property
|
||||
def pdf_pricing(self) -> PdfPagePricing:
|
||||
return pdf_page_pricing(self._model)
|
||||
|
||||
|
||||
def _is_openai_chat_model(model: str) -> bool:
|
||||
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
|
||||
|
||||
|
||||
def _openai_deployment_name(model: str) -> str:
|
||||
name: Final = model.rpartition("/")[2]
|
||||
return name.split(":")[1] if name.startswith("ft:") else name
|
||||
|
||||
|
||||
def _openai_model_family(model: str) -> str | None:
|
||||
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
|
||||
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
|
||||
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
|
||||
return None if listed is None or _is_listed_non_chat_model(listed) else listed
|
||||
|
||||
|
||||
def _untyped_cost_map() -> object:
|
||||
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
|
||||
|
||||
|
||||
def _cost_map_entry(model: str) -> Mapping[str, object]:
|
||||
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
|
||||
return _pdf_dictionary(cost_map.get(model)) or {}
|
||||
|
||||
|
||||
def _is_azure_openai_chat_model(model: str) -> bool:
|
||||
name: Final = _openai_deployment_name(model)
|
||||
entry: Final = _cost_map_entry(f"azure/{name}")
|
||||
return (
|
||||
entry.get("litellm_provider") == "azure"
|
||||
and entry.get("mode") in OPENAI_CHAT_MODES
|
||||
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
|
||||
)
|
||||
|
||||
|
||||
def _is_openai_or_azure_model(model: str) -> bool:
|
||||
if _is_listed_non_chat_model(_openai_deployment_name(model)):
|
||||
return False
|
||||
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
|
||||
|
||||
|
||||
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
|
||||
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
|
||||
|
||||
|
||||
def reads_pdf_pages(model: str) -> bool:
|
||||
segments: Final = model.split("/")
|
||||
names: Final = ("/".join(segments[start:]) for start in range(len(segments)))
|
||||
stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names)
|
||||
return next((support for support in stated if support is not None), False) is True
|
||||
|
||||
|
||||
def token_counter(
|
||||
|
|
@ -842,7 +937,10 @@ def _count_messages(
|
|||
value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
params.pdf_pricing,
|
||||
)
|
||||
elif key == "provider_specific_fields":
|
||||
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
|
||||
elif key == "search_results" and isinstance(value, list):
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_search_results_text,
|
||||
|
|
@ -857,6 +955,25 @@ def _count_messages(
|
|||
return num_tokens
|
||||
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
|
||||
fields: Final = as_str_mapping(provider_specific_fields)
|
||||
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
block_fields: Final = (
|
||||
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
|
||||
)
|
||||
return tuple(
|
||||
content
|
||||
for block in block_fields
|
||||
if block is not None
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(content := block.get("content"), str)
|
||||
and content
|
||||
)
|
||||
|
||||
|
||||
def _count_extra(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
|
|
@ -986,7 +1103,7 @@ def _get_exact_count_function(
|
|||
|
||||
return count_tokens
|
||||
if tokenizer_json["type"] == "openai_tokenizer":
|
||||
encoding: Final = openai_tokenizer_encoding(model)
|
||||
encoding: Final = openai_tokenizer_encoding(model or "")
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return _encoding_count(encoding, text)
|
||||
|
|
@ -1006,6 +1123,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
return get_encoding(openai_tokenizer_encoding_name(model))
|
||||
|
||||
|
||||
def _is_listed_non_chat_model(model: str) -> bool:
|
||||
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
|
||||
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
|
||||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding.
|
||||
|
||||
|
|
@ -1021,7 +1143,9 @@ def openai_tokenizer_encoding_name(model: str) -> str:
|
|||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
|
||||
return _suffixed_variant_encoding_name(model_to_use) or (
|
||||
"cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
|
||||
)
|
||||
|
||||
|
||||
def unresolved_tokenizer_model(model: str) -> str:
|
||||
|
|
@ -1079,7 +1203,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
|
|||
|
||||
|
||||
def _openai_model_name(model: str) -> str | None:
|
||||
return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
|
||||
return (
|
||||
_listed_openai_model_name(model)
|
||||
or _tiktoken_o200k_model_name(model)
|
||||
or _versioned_openai_model_name(model)
|
||||
or _azure_openai_chat_model_name(model)
|
||||
)
|
||||
|
||||
|
||||
def _azure_openai_chat_model_name(model: str) -> str | None:
|
||||
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None
|
||||
|
||||
|
||||
_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
|
||||
|
|
@ -1249,7 +1382,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
|
|||
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
|
||||
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
|
||||
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
|
||||
|
||||
|
||||
def _pdf_value(container: Mapping[str, object], key: str) -> object:
|
||||
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
|
||||
|
||||
|
||||
_PdfXObjects: TypeAlias = Mapping[str, object] | None
|
||||
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfContentSummary:
|
||||
drawn_names: tuple[str, ...]
|
||||
draws_inline_image: bool
|
||||
|
||||
|
||||
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfWalk:
|
||||
frontier: tuple[_PdfDrawing, ...]
|
||||
seen: frozenset[tuple[int, int]]
|
||||
summaries: Mapping[int, _PdfContentSummary | None]
|
||||
draws: int
|
||||
|
||||
|
||||
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
|
||||
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
|
||||
if resources is None:
|
||||
return inherited
|
||||
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
|
||||
|
||||
|
||||
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
|
||||
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
|
||||
_PDF_XOBJECT_DRAW: Final = re.compile(
|
||||
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
|
||||
)
|
||||
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
|
||||
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
|
||||
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
|
||||
|
||||
|
||||
def _decoded(raw: bytes, charset: str) -> str | None:
|
||||
try:
|
||||
return raw.decode(charset)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _pdf_name(escaped: bytes) -> str:
|
||||
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
|
||||
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
|
||||
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
|
||||
|
||||
|
||||
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
|
||||
data: Final = content.get_data()
|
||||
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
|
||||
return _PdfContentSummary(
|
||||
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
|
||||
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
|
||||
from pypdf.generic import StreamObject
|
||||
|
||||
if not isinstance(form, StreamObject):
|
||||
return None
|
||||
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
|
||||
|
||||
|
||||
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
|
||||
summary, xobjects = drawing
|
||||
if xobjects is None:
|
||||
return ()
|
||||
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
|
||||
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
|
||||
|
||||
|
||||
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
|
||||
xobject, inherited = use
|
||||
return id(xobject), id(_pdf_xobjects(xobject, inherited))
|
||||
|
||||
|
||||
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
|
||||
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
|
||||
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
|
||||
return None
|
||||
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
|
||||
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
|
||||
return None
|
||||
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
|
||||
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
|
||||
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
|
||||
summaries: Final = MappingProxyType(
|
||||
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
|
||||
)
|
||||
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
|
||||
return _PdfWalk(
|
||||
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
|
||||
seen=walk.seen | frozenset(forms),
|
||||
summaries=summaries,
|
||||
draws=draws,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
|
||||
contents: Final = page.get_contents()
|
||||
page_dictionary: Final = _pdf_dictionary(page)
|
||||
if contents is None or page_dictionary is None:
|
||||
return False
|
||||
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
|
||||
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
|
||||
seen=frozenset(),
|
||||
summaries=MappingProxyType({}),
|
||||
draws=0,
|
||||
)
|
||||
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
|
||||
if not walk.frontier:
|
||||
return False
|
||||
if any(summary.draws_inline_image for summary, _ in walk.frontier):
|
||||
return True
|
||||
if (next_walk := _pdf_next_walk(walk)) is None:
|
||||
return True
|
||||
walk = next_walk
|
||||
return bool(walk.frontier)
|
||||
|
||||
|
||||
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
if _pdf_page_draws_raster_image(page):
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
text: Final = page.extract_text() or ""
|
||||
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
|
||||
|
||||
|
||||
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
|
||||
float(page.mediabox.width), float(page.mediabox.height)
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
|
||||
match pdf_pricing:
|
||||
case "openai":
|
||||
return _openai_pdf_page_tokens(page, count_function)
|
||||
case "claude":
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(
|
||||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int | None:
|
||||
if not data_url.startswith(PDF_DATA_URL_PREFIX):
|
||||
return None
|
||||
try:
|
||||
|
|
@ -1258,12 +1549,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
|
|||
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
|
||||
return None
|
||||
try:
|
||||
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
return sum(
|
||||
count_function(page.extract_text() or "")
|
||||
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
|
||||
for page in reader.pages
|
||||
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
|
||||
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
)
|
||||
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
|
||||
return None
|
||||
|
|
@ -1273,8 +1562,9 @@ def _count_opaque_document_tokens(
|
|||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
|
||||
if pdf_tokens is not None:
|
||||
return pdf_tokens
|
||||
return calculate_img_tokens(
|
||||
|
|
@ -1289,6 +1579,7 @@ def _count_document_tokens(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
source: Final = document["source"]
|
||||
metadata_tokens: Final = sum(
|
||||
|
|
@ -1301,10 +1592,10 @@ def _count_document_tokens(
|
|||
if isinstance(content, str):
|
||||
return metadata_tokens + count_function(content)
|
||||
return metadata_tokens + _count_content_list(
|
||||
count_function, content, use_default_image_token_count, default_token_count
|
||||
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
|
||||
)
|
||||
return metadata_tokens + _count_opaque_document_tokens(
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1312,6 +1603,7 @@ def _count_file_tokens(
|
|||
file_value: object,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
|
|
@ -1321,7 +1613,9 @@ def _count_file_tokens(
|
|||
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
|
||||
if not isinstance(file_data, str) or not file_data:
|
||||
return name_tokens
|
||||
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
|
||||
return name_tokens + _count_opaque_document_tokens(
|
||||
file_data, count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
|
|
@ -1329,6 +1623,7 @@ def _count_anthropic_content(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""
|
||||
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
|
||||
|
|
@ -1363,6 +1658,7 @@ def _count_anthropic_content(
|
|||
field_value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif isinstance(field_value, dict):
|
||||
tokens += count_function(str(field_value))
|
||||
|
|
@ -1375,6 +1671,11 @@ def _count_anthropic_content(
|
|||
return tokens
|
||||
|
||||
|
||||
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
|
||||
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
|
||||
)
|
||||
|
||||
|
||||
def _count_content_list(
|
||||
count_function: TokenCounterFunction,
|
||||
content_list: str
|
||||
|
|
@ -1386,9 +1687,11 @@ def _count_content_list(
|
|||
| AnthropicMessagesImageParam
|
||||
| AnthropicMessagesDocumentParam
|
||||
| AnthropicSearchResultParam
|
||||
| CompactionBlock
|
||||
],
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""Recursively count tokens from a list of content blocks."""
|
||||
try:
|
||||
|
|
@ -1396,8 +1699,10 @@ def _count_content_list(
|
|||
for c in content_list:
|
||||
if isinstance(c, str):
|
||||
num_tokens += count_function(c)
|
||||
elif c["type"] == "text":
|
||||
num_tokens += count_function(str(c.get("text", "")))
|
||||
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
|
||||
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
|
||||
if block_text:
|
||||
num_tokens += count_function(block_text)
|
||||
elif c["type"] == "image_url":
|
||||
image_url = c.get("image_url")
|
||||
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
|
||||
|
|
@ -1413,12 +1718,14 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] == "file":
|
||||
num_tokens += _count_file_tokens(
|
||||
c.get("file"),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result", "search_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
|
|
@ -1426,13 +1733,8 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("thinking", "redacted_thinking"):
|
||||
# Claude extended thinking content block
|
||||
# Count the thinking text and skip the opaque blobs (signature, redacted data)
|
||||
thinking_text = str(c.get("thinking", ""))
|
||||
if thinking_text:
|
||||
num_tokens += count_function(thinking_text)
|
||||
elif c["type"] == "tool_reference":
|
||||
# Anthropic tool-search reference block: a lightweight pointer to
|
||||
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
|
||||
|
|
@ -1450,7 +1752,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"tool_reference, search_result)."
|
||||
f"compaction, tool_reference, search_result)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import json
|
|||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -28,6 +29,8 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _A2ATextPart(TypedDict, total=False):
|
||||
"""The subset of an A2A message part this handler reads text from."""
|
||||
|
|
@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
obj = item
|
||||
elif isinstance(item, str):
|
||||
try:
|
||||
obj = json.loads(item.strip())
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
obj = _JSON_OBJECT.validate_python(json.loads(item.strip()))
|
||||
except (json.JSONDecodeError, TypeError, ValidationError):
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ from copy import deepcopy
|
|||
from dataclasses import dataclass
|
||||
from itertools import chain, repeat
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
|
|
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
unappliable_request_rewrite,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
|
|
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
|
|||
text: str
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _WritableMessage(Protocol):
|
||||
@overload
|
||||
def get(self, key: str, /) -> object | None: ...
|
||||
|
|
@ -195,6 +197,140 @@ class _ToolCallShape:
|
|||
arguments: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _WebSearchString:
|
||||
block_idx: int
|
||||
result_idx: int | None
|
||||
field: str
|
||||
value: str
|
||||
|
||||
|
||||
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
|
||||
choice: Final = response.choices[0] if response.choices else None
|
||||
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
|
||||
blocks: Final = (
|
||||
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
|
||||
if fields is not None and isinstance(fields.get("web_search_results"), list)
|
||||
else ()
|
||||
)
|
||||
return tuple(
|
||||
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
|
||||
)
|
||||
|
||||
|
||||
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
|
||||
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
|
||||
|
||||
|
||||
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
|
||||
fields: Final = as_str_mapping(result)
|
||||
if fields is None or fields.get("type") != "web_search_result":
|
||||
return ()
|
||||
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
|
||||
return tuple(
|
||||
_WebSearchString(block_idx, result_idx, field, value)
|
||||
for field, value in values
|
||||
if isinstance(value, str) and value
|
||||
)
|
||||
|
||||
|
||||
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
|
||||
block_id: Final = block.get("id")
|
||||
match block.get("type"):
|
||||
case "server_tool_use":
|
||||
return True
|
||||
case "tool_use":
|
||||
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _web_search_block_strings(
|
||||
block_idx: int, block: Mapping[str, object], *, with_query: bool
|
||||
) -> tuple[_WebSearchString, ...]:
|
||||
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
|
||||
``web_search_tool_result`` result's title and url."""
|
||||
match block.get("type"):
|
||||
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
|
||||
tool_input: Final = as_str_mapping(block.get("input"))
|
||||
query: Final = None if tool_input is None else tool_input.get("query")
|
||||
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
|
||||
case "web_search_tool_result":
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_result_strings(block_idx, result_idx, result)
|
||||
for result_idx, result in enumerate(_web_search_results(block))
|
||||
)
|
||||
)
|
||||
case _:
|
||||
return ()
|
||||
|
||||
|
||||
def _rewritten_web_search_result(
|
||||
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
|
||||
) -> object:
|
||||
fields: Final = as_str_mapping(result)
|
||||
if fields is None:
|
||||
return result
|
||||
return {
|
||||
**fields,
|
||||
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
|
||||
}
|
||||
|
||||
|
||||
def _rewritten_web_search_block(
|
||||
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
|
||||
) -> Mapping[str, object]:
|
||||
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
|
||||
if _is_server_tool_use(block):
|
||||
query: Final = by_location.get((None, "query"))
|
||||
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
|
||||
return block if query is None else {**block, "input": {**tool_input, "query": query}}
|
||||
if not isinstance(block.get("content"), list):
|
||||
return block
|
||||
return {
|
||||
**block,
|
||||
"content": [
|
||||
_rewritten_web_search_result(result_idx, result, by_location)
|
||||
for result_idx, result in enumerate(_web_search_results(block))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
|
||||
block: Final = as_str_mapping(content_block)
|
||||
if block is None or not isinstance(content_block, _WritableMessage):
|
||||
return
|
||||
rewritten: Final = _rewritten_web_search_block(block, rewrites)
|
||||
writable: Final = _as_writable(content_block)
|
||||
for key in ("input", "content"):
|
||||
if key in rewritten:
|
||||
writable[key] = rewritten[key]
|
||||
|
||||
|
||||
def _split_web_search_texts(
|
||||
guardrailed: Sequence[str], web_search_count: int, reply_count: int
|
||||
) -> tuple[Sequence[str], Sequence[str]]:
|
||||
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
|
||||
return guardrailed[:split_at], guardrailed[split_at:]
|
||||
|
||||
|
||||
def _web_search_rewrites_by_block(
|
||||
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
|
||||
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
|
||||
rewrites: Final = tuple(
|
||||
(target, value)
|
||||
for target, value in zip(strings, guardrailed)
|
||||
if isinstance(value, str) and value != target.value
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
|
||||
for block_idx in {target.block_idx for target, _ in rewrites}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
|
||||
return (
|
||||
block.get("type") == "tool_use"
|
||||
|
|
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
|
|||
|
||||
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_DECODED_SSE_DATA: Final = TypeAdapter(object)
|
||||
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
|
||||
_RELEASED_TOOL_USE_STOP: Final = (
|
||||
b"event: message_delta\n"
|
||||
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
|
||||
|
|
@ -315,6 +452,51 @@ def _write_back_tool_use(
|
|||
block["name"] = shape.name
|
||||
|
||||
|
||||
def _streamed_query_strings(
|
||||
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
|
||||
) -> tuple[_WebSearchString, ...]:
|
||||
if len(tool_call_blocks) != len(tool_calls):
|
||||
return ()
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(
|
||||
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
|
||||
)
|
||||
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
target.block_idx: value
|
||||
for target, value in zip(strings, guardrailed)
|
||||
if isinstance(value, str) and value != target.value
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
|
||||
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
|
||||
if arguments is None:
|
||||
return tool_call
|
||||
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
|
||||
|
||||
|
||||
def _with_query_rewrites(
|
||||
tool_calls: tuple[_ToolCallShape, ...],
|
||||
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
|
||||
query_rewrites: Mapping[int, str],
|
||||
) -> tuple[_ToolCallShape, ...]:
|
||||
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
|
||||
return tool_calls
|
||||
return tuple(
|
||||
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
|
||||
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SSEFieldRewrite:
|
||||
"""One field of one nested section of a buffered SSE event, rewritten."""
|
||||
|
|
@ -349,14 +531,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...
|
|||
)
|
||||
|
||||
|
||||
class _AnthropicSSEDelta(TypedDict, total=False):
|
||||
type: ReadOnly[str]
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSETextDelta(TypedDict, total=False):
|
||||
type: ReadOnly[object]
|
||||
text: ReadOnly[str]
|
||||
stop_reason: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _AnthropicSSEEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_AnthropicSSEDelta]
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSETextDeltaEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_SSETextDelta]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSEStopDelta(TypedDict, total=False):
|
||||
stop_reason: ReadOnly[object]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSEStopDeltaEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_SSEStopDelta]
|
||||
|
||||
|
||||
_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent)
|
||||
_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent)
|
||||
_SSEEventT = TypeVar("_SSEEventT")
|
||||
|
||||
|
||||
def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None:
|
||||
try:
|
||||
return adapter.validate_python(json.loads(data_line))
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type)
|
||||
except ValidationError:
|
||||
verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type)
|
||||
return None
|
||||
|
||||
|
||||
class AnthropicMessagesHandler(BaseTranslation):
|
||||
|
|
@ -376,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
def post_call_hook_response(self, response: object) -> object:
|
||||
if not isinstance(response, ModelResponse):
|
||||
return response
|
||||
return self.adapter.translate_openai_response_to_anthropic(response)
|
||||
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
|
||||
result_blocks: Final = _streamed_web_search_result_blocks(response)
|
||||
if not result_blocks:
|
||||
return translated
|
||||
content: Final = translated.get("content") or ()
|
||||
return {**translated, "content": [*result_blocks, *content]}
|
||||
|
||||
@staticmethod
|
||||
def _build_streaming_usage_response(
|
||||
|
|
@ -1247,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
task_mappings,
|
||||
tool_calls_to_check,
|
||||
)
|
||||
block_fields: Final = tuple(map(self._output_block_fields, response_content))
|
||||
web_search_strings: Final = tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(content_idx, fields[1], with_query=True)
|
||||
for content_idx, fields in enumerate(block_fields)
|
||||
if fields is not None
|
||||
)
|
||||
)
|
||||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
if texts_to_check or tool_calls_to_check or web_search_strings:
|
||||
request_data = self._prepare_request_data(
|
||||
request_data,
|
||||
response,
|
||||
|
|
@ -1258,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
)
|
||||
|
||||
inputs: Final = self._build_guardrail_inputs(
|
||||
texts_to_check,
|
||||
[*(target.value for target in web_search_strings), *texts_to_check],
|
||||
images_to_check,
|
||||
tool_calls_to_check,
|
||||
response,
|
||||
|
|
@ -1271,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
|
||||
web_search_rewrites, reply_rewrites = _split_web_search_texts(
|
||||
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
|
||||
)
|
||||
|
||||
# Step 3: Map guardrail responses back to original response structure
|
||||
await self._apply_guardrail_responses_to_output(
|
||||
response=response,
|
||||
responses=guardrailed_texts,
|
||||
responses=list(reply_rewrites),
|
||||
task_mappings=task_mappings,
|
||||
)
|
||||
self._write_back_web_search_strings(
|
||||
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
|
||||
|
||||
|
|
@ -1324,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
string_so_far = first_choice.message.content
|
||||
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
|
||||
guardrail_inputs: Final = GenericGuardrailAPIInputs()
|
||||
if string_so_far:
|
||||
guardrail_inputs["texts"] = [string_so_far]
|
||||
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
|
||||
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
|
||||
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
|
||||
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
|
||||
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
|
||||
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
|
||||
if texts_to_check:
|
||||
guardrail_inputs["texts"] = texts_to_check
|
||||
if tool_calls_list:
|
||||
guardrail_inputs["tool_calls"] = tool_calls_list
|
||||
|
||||
|
|
@ -1348,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
responses_so_far, request_data
|
||||
)
|
||||
raise
|
||||
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
|
||||
web_search_rewrites, reply_rewrites = _split_web_search_texts(
|
||||
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
|
||||
)
|
||||
if (
|
||||
deliver_ended_stream_rewrites
|
||||
and isinstance(string_so_far, str)
|
||||
and string_so_far
|
||||
and guardrailed_texts
|
||||
and guardrailed_texts[0] != string_so_far
|
||||
and reply_rewrites
|
||||
and reply_rewrites[0] != string_so_far
|
||||
):
|
||||
self._write_ended_stream_text_rewrite(
|
||||
responses_so_far,
|
||||
guardrailed_texts[0],
|
||||
reply_rewrites[0],
|
||||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
if deliver_ended_stream_rewrites:
|
||||
self._write_ended_stream_web_search_rewrites(
|
||||
responses_so_far,
|
||||
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
|
||||
)
|
||||
if deliver_ended_stream_rewrites:
|
||||
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
|
||||
self._write_ended_stream_tool_call_rewrites(
|
||||
responses_so_far,
|
||||
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
|
||||
post_guardrail_tool_calls=_tool_call_shapes(
|
||||
returned_tool_calls
|
||||
if isinstance(returned_tool_calls, list)
|
||||
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
|
||||
else tool_calls_list or ()
|
||||
post_guardrail_tool_calls=_with_query_rewrites(
|
||||
_tool_call_shapes(
|
||||
returned_tool_calls
|
||||
if isinstance(returned_tool_calls, list)
|
||||
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
|
||||
else tool_calls_list or ()
|
||||
),
|
||||
tool_call_blocks,
|
||||
_query_rewrites(
|
||||
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
|
||||
),
|
||||
),
|
||||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
|
|
@ -1425,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
request_data["litellm_metadata"] = user_metadata
|
||||
return request_data
|
||||
|
||||
@staticmethod
|
||||
def _write_back_web_search_strings(
|
||||
response_content: Sequence[object],
|
||||
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
|
||||
) -> None:
|
||||
for block_idx, rewrites in rewrites_by_block.items():
|
||||
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
|
||||
|
||||
@staticmethod
|
||||
def _get_response_content(response: object) -> Sequence[object]:
|
||||
"""Extract content list from a dict or object response."""
|
||||
|
|
@ -1547,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
) -> None:
|
||||
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
|
||||
buffered chunks in place: the rebuilt response lists tool calls in the
|
||||
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
|
||||
on the nth block, its first ``input_json_delta`` carrying the full rewritten
|
||||
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
|
||||
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
|
||||
arguments, every later one blanked, and ``content_block_start`` carrying the
|
||||
rewritten name. Blocks that do not line up with the rebuilt tool calls make
|
||||
the rewrite undeliverable, so the pipeline executor discards it and releases
|
||||
the original chunks."""
|
||||
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
|
||||
return
|
||||
block_indices: Final = tuple(
|
||||
index
|
||||
for item in responses_so_far
|
||||
for event in cls._iter_sse_events(item)
|
||||
if event.get("type") == "content_block_start"
|
||||
and isinstance(block := event.get("content_block"), Mapping)
|
||||
and block.get("type") == "tool_use"
|
||||
and isinstance(index := event.get("index"), int)
|
||||
)
|
||||
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
|
||||
if len(block_indices) != len(post_guardrail_tool_calls):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
raise UndeliverableStreamRewrite(
|
||||
guardrail_name,
|
||||
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
|
||||
f"{len(block_indices)} tool_use blocks",
|
||||
f"{len(block_indices)} tool_use or server_tool_use blocks",
|
||||
)
|
||||
rewrites_by_block: Final = MappingProxyType(
|
||||
{
|
||||
|
|
@ -1610,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
|
||||
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
|
||||
result_blocks: Final = (
|
||||
(index, block)
|
||||
for event in events
|
||||
if event.get("type") == "content_block_start"
|
||||
and isinstance(index := event.get("index"), int)
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
)
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _write_ended_stream_web_search_rewrites(
|
||||
cls,
|
||||
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
|
||||
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
|
||||
) -> None:
|
||||
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
|
||||
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
|
||||
if not rewrites_by_block:
|
||||
return
|
||||
|
||||
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
|
||||
index: Final = event.get("index")
|
||||
block: Final = as_str_mapping(event.get("content_block"))
|
||||
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
|
||||
if event.get("type") != "content_block_start" or block is None or rewrites is None:
|
||||
return None
|
||||
return _SSEFieldRewrite(
|
||||
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
|
||||
)
|
||||
|
||||
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
|
||||
|
||||
@classmethod
|
||||
def _streamed_tool_call_blocks(
|
||||
cls, responses_so_far: Sequence[object]
|
||||
) -> tuple[tuple[int, Mapping[str, object]], ...]:
|
||||
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
|
||||
return tuple(
|
||||
(index, block)
|
||||
for event in events
|
||||
if event.get("type") == "content_block_start"
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
and isinstance(block_type := block.get("type"), str)
|
||||
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
|
||||
and isinstance(index := event.get("index"), int)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
|
||||
"""The id of each buffered ``tool_use`` block, by its content block index."""
|
||||
|
|
@ -1712,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
|
||||
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
|
||||
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
|
||||
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
|
||||
return StreamingScanKey(
|
||||
texts=(self.get_streaming_string_so_far(responses_so_far),),
|
||||
texts=(*(target.value for target in result_strings), streamed_text),
|
||||
tool_calls=tool_use_fingerprints if stream_ended else (),
|
||||
stream_ended=stream_ended,
|
||||
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,
|
||||
|
|
@ -1806,13 +2107,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Only process content_block_delta events
|
||||
if event_type == "content_block_delta" and data_line:
|
||||
try:
|
||||
data: _AnthropicSSEEvent = json.loads(data_line)
|
||||
delta: _AnthropicSSEDelta = data.get("delta", {})
|
||||
if delta.get("type") == "text_delta":
|
||||
text += delta.get("text", "")
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
|
||||
data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line)
|
||||
delta: _SSETextDelta = {} if data is None else data.get("delta", {})
|
||||
if delta.get("type") == "text_delta":
|
||||
text += delta.get("text", "")
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error extracting text from SSE: %s", e)
|
||||
|
|
@ -1869,14 +2167,11 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Check for message_delta event with stop_reason
|
||||
if event_type == "message_delta" and data_line:
|
||||
try:
|
||||
data: _AnthropicSSEEvent = json.loads(data_line)
|
||||
delta: _AnthropicSSEDelta = data.get("delta", {})
|
||||
stop_reason: str | None = delta.get("stop_reason")
|
||||
if stop_reason is not None:
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
|
||||
data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line)
|
||||
delta: _SSEStopDelta = {} if data is None else data.get("delta", {})
|
||||
stop_reason: object = delta.get("stop_reason")
|
||||
if stop_reason is not None:
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e)
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ import re
|
|||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -24,6 +25,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
|||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_JSON_VALUE: Final = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
|
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
|
|||
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
|
||||
|
||||
|
||||
def _message_blocks(message: object) -> Sequence[object]:
|
||||
fields: Final = as_str_mapping(message)
|
||||
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
|
||||
return blocks or ()
|
||||
|
||||
|
||||
def _is_signed_compaction_block(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
if fields is None or fields.get("type") != "compaction":
|
||||
return False
|
||||
signature: Final = fields.get("signature")
|
||||
return isinstance(signature, str) and bool(signature)
|
||||
|
||||
|
||||
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
|
||||
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
|
||||
)
|
||||
|
||||
|
||||
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
|
||||
if not isinstance(context_management, list):
|
||||
return context_management
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
|
||||
[entry for entry in context_management if isinstance(entry, dict)]
|
||||
)
|
||||
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
|
||||
|
||||
|
||||
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
|
||||
mapping: Final = as_str_mapping(spec)
|
||||
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
|
||||
|
||||
|
||||
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
|
||||
spec: Final = anthropic_context_management(context_management)
|
||||
return spec if _context_management_edits(spec) else None
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
|
||||
edits: Final = _context_management_edits(anthropic_context_management(context_management))
|
||||
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
any(edit_type != "compact_20260112" for edit_type in edit_types),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
|
|
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
context_management=context_management,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
|
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
verbose_logger.debug("Making request to: %s", endpoint_url)
|
||||
|
||||
# Get required headers
|
||||
headers: Final = self.get_count_tokens_headers(auth_header)
|
||||
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
|
||||
|
||||
# Use LiteLLM's async httpx client
|
||||
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
context_management=context_management.spec if context_management is not None else None,
|
||||
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
carries_signed_compaction_block,
|
||||
context_management_betas,
|
||||
context_management_with_edits,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
|
||||
"""
|
||||
|
|
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
|
|||
drop_custom=drop_params,
|
||||
),
|
||||
),
|
||||
("context_management", context_management_with_edits(context_management)),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
|
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
|
|||
)
|
||||
)
|
||||
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
|
||||
def get_count_tokens_headers(
|
||||
self,
|
||||
auth_header: Mapping[str, str],
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
context_management: JsonValue = None,
|
||||
) -> dict[str, str]:
|
||||
"""The count-tokens headers around a resolved Anthropic auth header
|
||||
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
|
||||
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
|
||||
merges with the token-counting beta instead of replacing it."""
|
||||
merges with the token-counting beta instead of replacing it. A history that carries a
|
||||
signed compaction block also needs the compaction beta, or Anthropic rejects the block
|
||||
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
|
||||
compaction_betas: Final = (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
|
||||
if carries_signed_compaction_block(messages)
|
||||
else ()
|
||||
)
|
||||
request_betas: Final = (
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
*compaction_betas,
|
||||
*context_management_betas(context_management),
|
||||
)
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
**auth_header,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
),
|
||||
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
|
||||
}
|
||||
|
||||
def validate_request(
|
||||
|
|
|
|||
|
|
@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
request_data["output_config"] = extra_kwargs["output_config"]
|
||||
|
||||
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
|
||||
api_base: Final = extra_kwargs.get("api_base")
|
||||
(
|
||||
openai_request,
|
||||
tool_name_mapping,
|
||||
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
|
||||
request_data,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
|
|||
TOOL_NAME_HASH_LENGTH: Final = 8
|
||||
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
|
||||
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
|
||||
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
|
||||
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
|
|
@ -255,7 +256,12 @@ class AnthropicAdapter:
|
|||
return result
|
||||
|
||||
def translate_completion_input_params_with_tool_mapping(
|
||||
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
|
||||
self,
|
||||
kwargs,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
drop_params: bool = True,
|
||||
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
|
||||
"""
|
||||
Translate Anthropic request params to OpenAI format, returning tool name mapping.
|
||||
|
|
@ -289,6 +295,7 @@ class AnthropicAdapter:
|
|||
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
|
||||
anthropic_message_request=request_body,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
|
@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
def _supports_prompt_cache_key(
|
||||
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
|
||||
) -> bool:
|
||||
if not model or not custom_llm_provider:
|
||||
return False
|
||||
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
|
||||
|
|
@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
supported_params: Final = litellm.get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
return "prompt_cache_key" in (supported_params or ())
|
||||
if "prompt_cache_key" not in (supported_params or ()):
|
||||
return False
|
||||
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
|
||||
return True
|
||||
_, _, _, resolved_api_base = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
)
|
||||
mistral_config: Final = litellm.MistralConfig()
|
||||
return mistral_config.takes_prompt_cache_key_value(
|
||||
prompt_cache_key
|
||||
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
|
||||
|
||||
@staticmethod
|
||||
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
|
||||
|
|
@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_kwargs: ChatCompletionRequest,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> None:
|
||||
"""Translate metadata fields from Anthropic request to OpenAI request."""
|
||||
if "metadata" in anthropic_message_request:
|
||||
|
|
@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_kwargs["user"] = metadata["user_id"]
|
||||
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
|
||||
if prompt_cache_key is not None and self._supports_prompt_cache_key(
|
||||
anthropic_message_request.get("model"), custom_llm_provider
|
||||
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
|
||||
):
|
||||
new_kwargs["prompt_cache_key"] = prompt_cache_key
|
||||
|
||||
|
|
@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
preserve_midturn_system: bool = False,
|
||||
drop_params: bool = True,
|
||||
) -> tuple[ChatCompletionRequest, dict[str, str]]:
|
||||
|
|
@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
## CONVERT TOOL CHOICE
|
||||
self._translate_tool_choice_to_openai(
|
||||
|
|
|
|||
|
|
@ -12,11 +12,20 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
||||
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
|
||||
citations: Final = block.get("citations")
|
||||
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
|
||||
|
||||
|
||||
class FakeAnthropicMessagesStreamIterator:
|
||||
"""
|
||||
|
|
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
|
||||
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
|
||||
"""Build SSE chunks for a single content block."""
|
||||
chunks: Final = []
|
||||
chunks: Final = list[bytes]()
|
||||
block_type: Final = block_dict.get("type")
|
||||
|
||||
if block_type == "text":
|
||||
citations: Final = _citations_of(block_dict)
|
||||
content_block_start = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
"content_block": {"type": "text", "text": "", "citations": ()}
|
||||
if citations
|
||||
else {"type": "text", "text": ""},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
citation_deltas: Final = (
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "citations_delta", "citation": citation},
|
||||
}
|
||||
for citation in citations
|
||||
)
|
||||
chunks.extend(
|
||||
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
|
||||
)
|
||||
text: Final = block_dict.get("text", "")
|
||||
content_block_delta = {
|
||||
"type": "content_block_delta",
|
||||
|
|
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
|
||||
|
||||
elif block_type == "redacted_thinking":
|
||||
content_block_start = {
|
||||
elif block_type in ("tool_use", "server_tool_use"):
|
||||
empty_input: Final[Mapping[str, object]] = {}
|
||||
tool_use_start: Final = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {"type": "redacted_thinking"},
|
||||
"content_block": {**block_dict, "input": empty_input},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
|
||||
elif block_type == "tool_use":
|
||||
content_block_start = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {
|
||||
"type": "tool_use",
|
||||
"id": block_dict.get("id"),
|
||||
"name": block_dict.get("name"),
|
||||
"input": {},
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
|
||||
input_data: Final = block_dict.get("input", {})
|
||||
content_block_delta = {
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
|
||||
tool_use_delta: Final = {
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {
|
||||
"type": "input_json_delta",
|
||||
"partial_json": json.dumps(input_data),
|
||||
"partial_json": json.dumps(block_dict.get("input", {})),
|
||||
},
|
||||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
|
||||
|
||||
else:
|
||||
passthrough_start: Final = {
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from ...common_utils import (
|
|||
OBJECT_LIST_ADAPTER,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
|
|
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
if transformed_context_management is not None:
|
||||
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
|
||||
elif drops_unsupported_params(litellm_params.drop_params):
|
||||
anthropic_messages_optional_request_params.pop("context_management")
|
||||
|
||||
####### get required params for all anthropic messages requests ######
|
||||
# Lazy %s: the f-string previously stringified the entire messages
|
||||
|
|
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", ())
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
# the schema under output_config.format; the older top-level
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from ..openai.common_utils import (
|
|||
build_output_token_limit_response,
|
||||
is_output_token_limit_error,
|
||||
model_dump_provider_response,
|
||||
upstream_error_response,
|
||||
)
|
||||
from .common_utils import (
|
||||
AzureOpenAIError,
|
||||
|
|
@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=error_body,
|
||||
)
|
||||
|
|
@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=error_body,
|
||||
)
|
||||
|
|
@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
error_text = error_response.text
|
||||
raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_text,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
)
|
||||
|
||||
async def make_async_azure_httpx_request(
|
||||
self,
|
||||
|
|
@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
except Exception as e:
|
||||
error_code: Final[object] = getattr(e, "status_code", None)
|
||||
if error_code is not None:
|
||||
raise AzureOpenAIError(status_code=error_code, message=str(e))
|
||||
raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e))
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
|
||||
|
||||
from ...openai.common_utils import upstream_error_response
|
||||
from ...openai.completion.transformation import OpenAITextCompletionConfig
|
||||
from ..common_utils import AzureOpenAIError, BaseAzureLLM
|
||||
|
||||
|
|
@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
||||
async def acompletion(
|
||||
self,
|
||||
|
|
@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
||||
def streaming(
|
||||
self,
|
||||
|
|
@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM):
|
|||
error_response: Final[object] = getattr(e, "response", None)
|
||||
if error_headers is None and error_response:
|
||||
error_headers = getattr(error_response, "headers", None)
|
||||
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
|
||||
raise AzureOpenAIError(
|
||||
status_code=status_code,
|
||||
message=str(e),
|
||||
headers=error_headers,
|
||||
response=upstream_error_response(e),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.exceptions import ContentPolicyViolationError
|
||||
|
||||
|
||||
|
|
@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping:
|
|||
model: str,
|
||||
extra_information: str,
|
||||
original_exception: Exception,
|
||||
response: httpx.Response | None,
|
||||
) -> ContentPolicyViolationError:
|
||||
"""
|
||||
Create a content policy violation error
|
||||
|
|
@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping:
|
|||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
response=response,
|
||||
provider_specific_fields={
|
||||
# Preserve legacy key for backward compatibility.
|
||||
"innererror": inner_error,
|
||||
|
|
|
|||
|
|
@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
content_parts,
|
||||
convert_content_list_to_str,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
is_pdf_file_part,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
|
|
@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
|
|||
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
|
||||
from litellm.llms.xai.chat.transformation import XAIChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
|
||||
from litellm.utils import add_path_to_api_base, supports_tool_choice
|
||||
|
|
@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
|
|||
return any(_is_list_only_content_part(part) for part in content_parts(message))
|
||||
|
||||
|
||||
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
|
||||
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return ()
|
||||
try:
|
||||
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
|
||||
except ValidationError:
|
||||
return ()
|
||||
return tuple(part for part in parts if part.get("type") == "text")
|
||||
|
||||
|
||||
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
|
||||
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
|
||||
return message
|
||||
text_only: Final = {**message, "content": list(text_parts)}
|
||||
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
|
||||
|
||||
|
||||
_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
|
||||
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
|
||||
return None
|
||||
return PromptCacheBreakpoint(mode="explicit")
|
||||
|
||||
|
||||
def _part_with_foundry_breakpoint(part: object) -> object:
|
||||
if not isinstance(part, Mapping):
|
||||
return part
|
||||
fields: Final = _FIELDS.validate_python(part)
|
||||
if "prompt_cache_breakpoint" not in fields:
|
||||
return fields
|
||||
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
|
||||
|
||||
|
||||
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
|
||||
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
|
||||
parts: Final = content_parts(renamed) if drop_params else ()
|
||||
if not parts:
|
||||
return renamed
|
||||
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
|
||||
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
|
||||
|
||||
|
||||
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
|
||||
model_info: Final = litellm_params.get("model_info")
|
||||
if not isinstance(model_info, Mapping):
|
||||
return any(
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
|
||||
)
|
||||
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
|
||||
if isinstance(deployment_flag, bool):
|
||||
return deployment_flag
|
||||
return any(
|
||||
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
|
||||
for name in named_models
|
||||
)
|
||||
|
||||
|
||||
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
|
||||
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
|
||||
usage: Final = chunk.get("usage")
|
||||
|
|
@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
self,
|
||||
messages: list[AllMessageValues],
|
||||
model: str,
|
||||
reads_breakpoints: bool | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> list:
|
||||
"""
|
||||
- Azure AI Studio doesn't support content as a list. This handles:
|
||||
|
|
@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
Azure AI Foundry backends set additionalProperties=false and reject
|
||||
these with "Extra inputs are not permitted", which breaks multi-turn
|
||||
Anthropic-format clients that echo thinking blocks back as history.
|
||||
2. Transforms list content to a string.
|
||||
2. Transforms list content to a string, except on deployments that read OpenAI explicit
|
||||
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
|
||||
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
|
||||
content part can carry the breakpoint (a message-level one is silently ignored), so those
|
||||
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
|
||||
way the openai provider renames it. Every other part is dropped, exactly as the string
|
||||
conversion drops it, before the marks are renamed, so a message-level mark lands on the
|
||||
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
|
||||
shape Foundry takes, and a malformed one is dropped.
|
||||
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
|
||||
|
||||
Operates on a deep copy so the caller's messages keep their thinking blocks
|
||||
and provider metadata, which a fallback to another provider still needs.
|
||||
"""
|
||||
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
deployment_reads_breakpoints: Final = (
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
|
||||
if reads_breakpoints is None
|
||||
else reads_breakpoints
|
||||
)
|
||||
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
stripped_messages: Final = (
|
||||
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
|
||||
if deployment_reads_breakpoints
|
||||
else copied_messages
|
||||
)
|
||||
for message in stripped_messages:
|
||||
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
|
||||
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
|
||||
|
|
@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
if _has_list_only_content_part(message):
|
||||
continue
|
||||
|
||||
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
|
||||
message_dict["content"] = list(text_parts)
|
||||
continue
|
||||
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
|
|
@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
if extra_body and isinstance(extra_body, dict):
|
||||
optional_params.update(extra_body)
|
||||
optional_params.pop("max_retries", None)
|
||||
return super().transform_request(model, messages, optional_params, litellm_params, headers)
|
||||
return {
|
||||
"model": model,
|
||||
"messages": self._transform_messages(
|
||||
messages,
|
||||
model,
|
||||
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
|
||||
bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
),
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
|
|
@ -18,6 +23,67 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float)
|
||||
|
||||
|
||||
def _token_count_or_none(value: object) -> int | float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value)
|
||||
except ValidationError:
|
||||
verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent")
|
||||
return None
|
||||
if not math.isfinite(count):
|
||||
verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent")
|
||||
return None
|
||||
return count
|
||||
|
||||
|
||||
_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageUsageDetails(TypedDict, total=False):
|
||||
text_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageUsage(TypedDict, total=False):
|
||||
input_tokens_details: ReadOnly[NotRequired[object]]
|
||||
num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
input_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
output_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
num_output_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
output_image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
total_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageResponse(TypedDict, total=False):
|
||||
usage: ReadOnly[NotRequired[object]]
|
||||
|
||||
|
||||
_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails)
|
||||
_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage)
|
||||
_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None:
|
||||
if usage is None:
|
||||
return None
|
||||
try:
|
||||
return _MAI_RAW_USAGE_ADAPTER.validate_python(usage)
|
||||
except ValidationError:
|
||||
verbose_logger.warning("MAI image usage is not an object, counting it as absent")
|
||||
return None
|
||||
|
||||
|
||||
_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse)
|
||||
|
||||
|
||||
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
|
||||
|
||||
|
|
@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
return "maiimage" in model_normalized
|
||||
|
||||
@staticmethod
|
||||
def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]:
|
||||
def normalize_mai_image_usage(usage: object) -> Mapping[str, object]:
|
||||
"""Map Azure MAI usage fields to OpenAI ImageUsage schema."""
|
||||
if usage is None:
|
||||
raw_usage: Final = _raw_usage_or_none(usage)
|
||||
if raw_usage is None:
|
||||
return {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
|
|
@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
"total_tokens": 0,
|
||||
}
|
||||
|
||||
normalized_usage: Final = dict(usage)
|
||||
input_tokens_details = normalized_usage.get("input_tokens_details")
|
||||
if not isinstance(input_tokens_details, dict):
|
||||
input_tokens_details = {}
|
||||
normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage)
|
||||
input_tokens_details_value: Final = normalized_usage.get("input_tokens_details")
|
||||
input_tokens_details: Final[_MAIImageUsageDetails] = (
|
||||
_MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value)
|
||||
if isinstance(input_tokens_details_value, dict)
|
||||
else {}
|
||||
)
|
||||
|
||||
text_tokens = normalized_usage.get("num_input_text_tokens")
|
||||
if text_tokens is None:
|
||||
|
|
@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
if total_tokens is None:
|
||||
total_tokens = input_tokens + output_tokens
|
||||
|
||||
normalized_usage.update(
|
||||
{
|
||||
"input_tokens": input_tokens,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": image_tokens,
|
||||
"text_tokens": text_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
}
|
||||
)
|
||||
return normalized_usage
|
||||
return {
|
||||
**raw_usage,
|
||||
"input_tokens": input_tokens,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": image_tokens,
|
||||
"text_tokens": text_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "size"]
|
||||
|
|
@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
try:
|
||||
response: Final = raw_response.json()
|
||||
response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json())
|
||||
except Exception:
|
||||
raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
if "usage" in response:
|
||||
response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
|
||||
response: Final = (
|
||||
{
|
||||
**response_json,
|
||||
"usage": self.normalize_mai_image_usage(response_json["usage"]),
|
||||
}
|
||||
if "usage" in response_json
|
||||
else dict(response_json)
|
||||
)
|
||||
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import dataclasses
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
from typing import TYPE_CHECKING, Any, Literal, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
|
||||
from litellm.types.realtime import (
|
||||
RealtimeInputAudioTranscriptionUsage,
|
||||
|
|
@ -22,6 +24,26 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class RealtimeSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
def error_event(self, client_event_id: object) -> dict[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": self.code,
|
||||
"message": self.message,
|
||||
"param": self.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class RealtimeBackend(Protocol):
|
||||
async def __aenter__(self) -> object: ...
|
||||
|
||||
|
|
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
|
|||
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
|
||||
return None
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return False
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
return None
|
||||
|
||||
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping
|
|||
from typing import TYPE_CHECKING, Final, get_args, overload
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
|
||||
|
|
@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config
|
|||
from .cohere_transformation import BedrockCohereEmbeddingConfig
|
||||
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
|
@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
|
||||
async def _make_async_call(
|
||||
self,
|
||||
|
|
@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
|
||||
def _transform_response(
|
||||
self,
|
||||
|
|
@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
|
||||
# Parse response
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
else:
|
||||
raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import redact_string, verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
|
||||
|
|
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
|
||||
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
|
||||
from litellm.types.llms.bedrock import AwsAuthParams
|
||||
from litellm.types.llms.openai import OpenAIRealtimeEvents
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from ..common_utils import BedrockError
|
||||
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
|
||||
from .transformation import BedrockRealtimeConfig, requested_modalities
|
||||
|
||||
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
|
@ -256,23 +256,11 @@ async def _ack_session_update(
|
|||
async def _reject_session_update(
|
||||
client_ws: RealtimeClientWebSocket,
|
||||
parsed_client_message: Mapping[str, JsonValue],
|
||||
rejection: BedrockSessionUpdateRejection,
|
||||
rejection: RealtimeSessionUpdateRejection,
|
||||
) -> bool:
|
||||
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
|
||||
client_event_id: Final = parsed_client_message.get("event_id")
|
||||
error_event: Final = {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": rejection.code,
|
||||
"message": rejection.message,
|
||||
"param": rejection.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
try:
|
||||
await client_ws.send_text(json.dumps(error_event))
|
||||
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
|
||||
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
|
||||
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import math
|
|||
import uuid as uuid_lib
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter
|
||||
|
|
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
|
|||
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockLockedSessionSettings:
|
||||
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
|
||||
|
|
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
|
||||
"""The error for a session.update this session cannot apply, or None when it applies."""
|
||||
session_config: Final = json_message.get("session")
|
||||
if not isinstance(session_config, Mapping):
|
||||
|
|
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
|
||||
if malformed:
|
||||
name, expected = malformed[0]
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_type",
|
||||
message=(
|
||||
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
|
||||
|
|
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
)
|
||||
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
|
||||
if unplayable_voice is not None:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
|
||||
|
|
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
if not self.prompt_started:
|
||||
return None
|
||||
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
|
||||
param="session",
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
|
|||
def custom_llm_provider(self) -> str:
|
||||
return "bigmodel"
|
||||
|
||||
@property
|
||||
def prompts_unsupported_json_schema(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
|
||||
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
|
||||
|
||||
|
|
|
|||
95
litellm/llms/clf_ai_gateway/chat/transformation.py
Normal file
95
litellm/llms/clf_ai_gateway/chat/transformation.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
SUPPORTED_OPENAI_PARAMS: Final = (
|
||||
"stream",
|
||||
"stream_options",
|
||||
"frequency_penalty",
|
||||
"presence_penalty",
|
||||
"logit_bias",
|
||||
"logprobs",
|
||||
"top_logprobs",
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"n",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"seed",
|
||||
"response_format",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"user",
|
||||
"store",
|
||||
"service_tier",
|
||||
)
|
||||
|
||||
|
||||
class ClfAiGatewayConfig(OpenAIGPTConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "clf_ai_gateway"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return
|
||||
supports_reasoning: Final = litellm.supports_reasoning(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
extra: Final = ("reasoning_effort",) if supports_reasoning else ()
|
||||
return [*SUPPORTED_OPENAI_PARAMS, *extra]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
requested: Final = non_default_params.get("reasoning_effort")
|
||||
requested_mapping: Final = as_str_mapping(requested)
|
||||
effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested
|
||||
declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway")
|
||||
drop_enabled: Final = drop_params or litellm.drop_params is True
|
||||
if requested_mapping is not None and effort is None and not drop_enabled:
|
||||
raise UnsupportedParamsError(
|
||||
message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop "
|
||||
"unsupported openai params from the call, set `litellm.drop_params = True`",
|
||||
llm_provider="clf_ai_gateway",
|
||||
model=model,
|
||||
status_code=400,
|
||||
)
|
||||
drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared
|
||||
if drops_effort:
|
||||
verbose_logger.warning(
|
||||
"Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.",
|
||||
effort,
|
||||
model,
|
||||
declared,
|
||||
)
|
||||
others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"}
|
||||
normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others
|
||||
forwarded: Final = others if drops_effort else normalized
|
||||
mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict
|
||||
non_default_params=forwarded,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
return mapped
|
||||
|
||||
def get_openai_compatible_provider_info(
|
||||
self, api_base: str | None, api_key: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
resolved_api_base: Final = (
|
||||
api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1"
|
||||
)
|
||||
dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY")
|
||||
return resolved_api_base, dynamic_api_key
|
||||
|
|
@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
|
|||
),
|
||||
)
|
||||
|
||||
# Auto-send session setup if the provider requires it (e.g.
|
||||
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
|
||||
# Auto-send session setup at connect unless the provider builds it from the
|
||||
# client's first session.update (Gemini/Vertex AI Live accept exactly one
|
||||
# setup, so RealTimeStreaming sends theirs on the first client message).
|
||||
# Build the streaming handler first so a transcription guardrail's
|
||||
# auto-response disable can be folded into this one setup: Gemini
|
||||
# rejects a second setup, so a follow-up disable would be dropped
|
||||
# and the guardrail bypassed.
|
||||
_session_config: str | None = None
|
||||
if provider_config.requires_session_configuration():
|
||||
sends_setup_at_connect: Final = (
|
||||
provider_config.requires_session_configuration()
|
||||
and not provider_config.builds_setup_from_session_update()
|
||||
)
|
||||
if sends_setup_at_connect:
|
||||
_session_config = provider_config.session_configuration_request(model)
|
||||
if _session_config:
|
||||
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
||||
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
|
||||
|
|
@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object
|
|||
|
||||
from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
|
||||
def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data
|
||||
"""LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a
|
||||
|
|
@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
|||
def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
|
||||
if "application/json" not in raw_response.headers.get("content-type", ""):
|
||||
return TranscriptionResponse(text=raw_response.text)
|
||||
body: Final = raw_response.json()
|
||||
body: Final = _JSON_OBJECT.validate_python(raw_response.json())
|
||||
response: Final[TranscriptionResponse] = convert_to_model_response_object(
|
||||
response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from collections import OrderedDict
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,8 +16,9 @@ from litellm._uuid import uuid
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
OPENAI_TO_GEMINI_TTS_VOICES,
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
|
|||
|
||||
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
|
||||
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
|
||||
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
|
||||
|
||||
|
||||
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
|
||||
)
|
||||
|
||||
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
|
||||
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"achernar",
|
||||
"achird",
|
||||
"algenib",
|
||||
"algieba",
|
||||
"alnilam",
|
||||
"aoede",
|
||||
"autonoe",
|
||||
"callirrhoe",
|
||||
"charon",
|
||||
"despina",
|
||||
"enceladus",
|
||||
"erinome",
|
||||
"fenrir",
|
||||
"gacrux",
|
||||
"iapetus",
|
||||
"kore",
|
||||
"laomedeia",
|
||||
"leda",
|
||||
"orus",
|
||||
"puck",
|
||||
"pulcherrima",
|
||||
"rasalgethi",
|
||||
"sadachbia",
|
||||
"sadaltager",
|
||||
"schedar",
|
||||
"sulafat",
|
||||
"umbriel",
|
||||
"vindemiatrix",
|
||||
"zephyr",
|
||||
"zubenelgenubi",
|
||||
}
|
||||
)
|
||||
|
||||
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
|
||||
|
||||
def _is_undocumented_live_voice(voice: object) -> bool:
|
||||
return (
|
||||
isinstance(voice, str)
|
||||
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
|
||||
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
|
||||
)
|
||||
|
||||
|
||||
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
|
||||
"""Build the Gemini Live speechConfig for a client-requested voice.
|
||||
|
||||
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
|
||||
the session on an unknown voice, so they are dropped with a warning and
|
||||
the model keeps its default voice. Every other name is forwarded verbatim.
|
||||
OpenAI stock voice names have no Gemini equivalent, so they are always
|
||||
dropped, and so is any name outside Google's documented Live voices when
|
||||
``drop_undocumented`` is set, keeping the model's default voice. Every
|
||||
other name is forwarded verbatim.
|
||||
"""
|
||||
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
if not isinstance(voice, str):
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
|
||||
"dropping it so the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
if drop_undocumented and _is_undocumented_live_voice(voice):
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
|
||||
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
|
||||
|
||||
|
|
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
|
|||
return max(len(data) * 3 // 4 - padding, 0)
|
||||
|
||||
|
||||
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _chat_completion_tool(tool: object) -> dict[str, object]:
|
||||
fields: Final = as_str_mapping(tool) or {}
|
||||
if fields.get("type") != "function" or "function" in fields:
|
||||
return {**fields}
|
||||
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
|
||||
|
||||
|
||||
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
|
||||
parts: Final = system_instruction.get("parts", []) if system_instruction else []
|
||||
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
|
||||
return "\n".join(texts) if texts else None
|
||||
|
||||
|
||||
class GeminiRealtimeConfig(BaseRealtimeConfig):
|
||||
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, drop_params: bool = False):
|
||||
super().__init__()
|
||||
self.drop_params = drop_params
|
||||
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
|
||||
# Gemini Live sometimes emits usageMetadata in a standalone frame between
|
||||
# turns; buffer it here so the next response.done carries the token counts.
|
||||
self._pending_usage_metadata: dict | None = None
|
||||
self._unbilled_input_audio_bytes: int = 0
|
||||
self._manual_activity_open: bool = False
|
||||
|
||||
def is_setup_message(self, msg_obj: dict) -> bool:
|
||||
return "setup" in msg_obj
|
||||
|
|
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
|
||||
return True
|
||||
|
||||
def _rejects_undocumented_voices(self, model: str) -> bool:
|
||||
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
|
||||
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
|
||||
|
||||
@staticmethod
|
||||
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
|
||||
detail_mapping: Final = as_str_mapping(details)
|
||||
|
|
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
|
||||
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
|
||||
return ()
|
||||
self._manual_activity_open = True
|
||||
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
|
||||
return (json.dumps({"realtimeInput": activity_start}),)
|
||||
|
||||
def _end_manual_activity(self) -> tuple[str, ...]:
|
||||
if not self._manual_activity_open:
|
||||
return ()
|
||||
self._manual_activity_open = False
|
||||
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
|
||||
return (json.dumps({"realtimeInput": activity_end}),)
|
||||
|
||||
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
|
||||
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
|
||||
if self._manual_turn_detection_enabled(session_configuration_request):
|
||||
realtime_input_dict: BidiGenerateContentRealtimeInput = {
|
||||
"activityEnd": True,
|
||||
}
|
||||
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
|
||||
else:
|
||||
realtime_input_dict = {"audioStreamEnd": True}
|
||||
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
|
||||
return [json.dumps({"realtimeInput": realtime_input_dict})]
|
||||
return list(self._end_manual_activity())
|
||||
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
|
||||
return [json.dumps({"realtimeInput": audio_stream_end})]
|
||||
|
||||
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
|
||||
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
|
||||
|
|
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"voice",
|
||||
]
|
||||
|
||||
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
|
||||
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
|
||||
if "generationConfig" not in optional_params:
|
||||
optional_params["generationConfig"] = {}
|
||||
for key, value in non_default_params.items():
|
||||
|
|
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
|
||||
elif key == "temperature":
|
||||
optional_params["generationConfig"]["temperature"] = value
|
||||
elif key == "max_response_output_tokens":
|
||||
elif key == "max_response_output_tokens" and isinstance(value, int):
|
||||
optional_params["generationConfig"]["maxOutputTokens"] = value
|
||||
elif key == "modalities":
|
||||
optional_params["generationConfig"]["responseModalities"] = [
|
||||
modality.upper() for modality in cast(list[str], value)
|
||||
]
|
||||
optional_params["generationConfig"]["responseModalities"] = list(
|
||||
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
|
||||
)
|
||||
elif key == "tools":
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
|
|
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
vertex_gemini_config = VertexGeminiConfig()
|
||||
# Tools should be at the top level of setup, not inside generationConfig
|
||||
optional_params["tools"] = vertex_gemini_config.map_function(
|
||||
value=value, optional_params=optional_params
|
||||
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
|
||||
optional_params=optional_params,
|
||||
)
|
||||
elif key == "input_audio_transcription" and value is not None:
|
||||
optional_params["inputAudioTranscription"] = {}
|
||||
elif key == "turn_detection" and value is not None:
|
||||
value_typed = cast(OpenAIRealtimeTurnDetection, value)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
continue
|
||||
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
|
||||
if transformed_audio_activity_config:
|
||||
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
|
||||
automaticActivityDetection=transformed_audio_activity_config
|
||||
)
|
||||
elif key == "turn_detection":
|
||||
realtime_input_config = self._realtime_input_config(cast(object, value))
|
||||
if realtime_input_config is not None:
|
||||
optional_params["realtimeInputConfig"] = realtime_input_config
|
||||
elif key == "voice":
|
||||
speech_config = _gemini_live_speech_config(value)
|
||||
speech_config = _gemini_live_speech_config(
|
||||
value, self.drop_params and self._rejects_undocumented_voices(model)
|
||||
)
|
||||
if speech_config:
|
||||
optional_params["generationConfig"]["speechConfig"] = speech_config
|
||||
if len(optional_params["generationConfig"]) == 0:
|
||||
optional_params.pop("generationConfig")
|
||||
return optional_params
|
||||
|
||||
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
|
||||
if turn_detection is None:
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
|
||||
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
return None
|
||||
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
|
||||
if not transformed_audio_activity_config:
|
||||
return None
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
|
||||
|
||||
@staticmethod
|
||||
def _extract_turn_detection(session: dict) -> dict | None:
|
||||
"""Extract turn_detection from a session.update payload.
|
||||
|
|
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
if isinstance(input_cfg, dict):
|
||||
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
|
||||
normalized["input_audio_transcription"] = input_cfg["transcription"]
|
||||
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
|
||||
normalized["turn_detection"] = input_cfg["turn_detection"]
|
||||
output_cfg: Final = audio.get("output")
|
||||
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
|
||||
normalized["voice"] = output_cfg["voice"]
|
||||
|
|
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _model_cost_entry(model: str) -> dict:
|
||||
def _model_cost_entry(model: str) -> Mapping[str, object]:
|
||||
entry = litellm.model_cost.get(model)
|
||||
if entry is None:
|
||||
stripped: Final = model.split("/", 1)[-1]
|
||||
|
|
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
|
||||
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
|
||||
normalized: Final = tuple(
|
||||
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
|
||||
def _requested_response_modalities(value: object) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value.upper(),)
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return ()
|
||||
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
|
||||
|
||||
@staticmethod
|
||||
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
|
||||
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if unknown:
|
||||
return unknown
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
|
||||
return ("TEXT",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
|
||||
return ("AUDIO",)
|
||||
return ()
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
if self.drop_params:
|
||||
return None
|
||||
if setup_sent:
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message=(
|
||||
"Gemini Live accepts one session configuration per connection, and this session already sent "
|
||||
"its configuration, so this session.update was not applied. Put every setting in session.update "
|
||||
"before the first conversation content, or set drop_params to drop it instead"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
|
||||
|
||||
@staticmethod
|
||||
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
|
||||
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
|
||||
)
|
||||
if not unsupported:
|
||||
return None
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.{param}': {model} cannot produce "
|
||||
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
|
||||
),
|
||||
param=f"session.{param}",
|
||||
)
|
||||
|
||||
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
|
||||
output_voice: Final = audio_output.get("voice")
|
||||
voice: Final = output_voice or session.get("voice")
|
||||
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
|
||||
return None
|
||||
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
|
||||
"Set drop_params to drop it instead"
|
||||
),
|
||||
param=param,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
|
||||
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
|
||||
text-only ones such as transcribe-live), then keep a single modality
|
||||
because Gemini Live rejects a setup naming two."""
|
||||
return GeminiRealtimeConfig._single_response_modality(
|
||||
GeminiRealtimeConfig._swap_unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
normalized: Final = tuple(str(modality).upper() for modality in modalities)
|
||||
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if len(known) < len(normalized):
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
|
||||
model,
|
||||
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
|
||||
)
|
||||
return known or (GeminiRealtimeConfig._default_response_modality(model),)
|
||||
|
||||
@staticmethod
|
||||
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
|
||||
model,
|
||||
)
|
||||
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
|
||||
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if len(modalities) <= 1:
|
||||
return modalities
|
||||
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
|
||||
chosen,
|
||||
", ".join(modalities),
|
||||
)
|
||||
return (chosen,)
|
||||
|
||||
@staticmethod
|
||||
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
|
||||
generation_config: Final = setup.get("generationConfig")
|
||||
|
|
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# would be silently dropped because ``map_openai_params`` only
|
||||
# recognises the flat OpenAI-beta key names.
|
||||
session_payload = self._normalize_session_payload_for_mapping(session_payload)
|
||||
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
|
||||
new_overrides: Final = self.map_openai_params(
|
||||
optional_params={}, non_default_params=session_payload, model=model
|
||||
)
|
||||
|
||||
if session_configuration_request is None:
|
||||
generation_config: Final = new_overrides.setdefault("generationConfig", {})
|
||||
|
|
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
message_str = str(message)
|
||||
raise ValueError(f"Invalid JSON message: {message_str}")
|
||||
|
||||
messages: Final[list[str]] = []
|
||||
msg_type: Final = json_message.get("type")
|
||||
|
||||
if msg_type == "session.update":
|
||||
return self._handle_session_update(json_message, model, session_configuration_request)
|
||||
|
||||
if msg_type == "response.create":
|
||||
return [] # Gemini responds automatically; nothing to forward
|
||||
return list(self._end_manual_activity())
|
||||
|
||||
if msg_type == "conversation.item.create":
|
||||
return self._handle_conversation_item(json_message)
|
||||
|
|
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
|
||||
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
|
||||
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
|
||||
messages.append(gemini_msg)
|
||||
return messages
|
||||
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
|
||||
|
||||
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
|
||||
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
|
||||
|
|
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
logging_session_id: str,
|
||||
session_configuration_request: str | None = None,
|
||||
) -> OpenAIRealtimeStreamSessionEvents:
|
||||
if session_configuration_request:
|
||||
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
|
||||
else:
|
||||
session_configuration_request_dict = {}
|
||||
session_configuration_request_dict: Final = _parse_setup(
|
||||
session_configuration_request or self.session_configuration_request(model)
|
||||
)
|
||||
|
||||
_model: Final = session_configuration_request_dict.get("model") or model
|
||||
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
|
||||
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
|
||||
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
|
||||
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
|
||||
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
|
||||
session: Final = OpenAIRealtimeStreamSession(
|
||||
id=logging_session_id,
|
||||
modalities=_modalities,
|
||||
)
|
||||
if _system_instruction is not None and isinstance(_system_instruction, str):
|
||||
session["instructions"] = _system_instruction
|
||||
if instructions is not None:
|
||||
session["instructions"] = instructions
|
||||
if _model is not None and isinstance(_model, str):
|
||||
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
|
||||
if "/models/" in _model:
|
||||
|
|
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
|
||||
return not litellm.gemini_live_defer_setup
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return True
|
||||
|
||||
def session_configuration_request(self, model: str) -> str:
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/
|
|||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from itertools import accumulate, count
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
|
|
@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
|
||||
from litellm.types.llms.mistral import (
|
||||
MistralCacheKeyParams,
|
||||
MistralFilePart,
|
||||
MistralReasoningEffortParams,
|
||||
MistralRequestRouting,
|
||||
MistralTextBlock,
|
||||
MistralThinkingBlock,
|
||||
MistralToolCallMessage,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
from litellm.utils import convert_to_model_response_object, supports_reasoning
|
||||
|
|
@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
|
|||
|
||||
|
||||
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
|
||||
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
|
||||
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
|
||||
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
|
||||
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
|
||||
|
||||
|
||||
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
|
||||
requested: Final = as_str_mapping(reasoning_effort)
|
||||
if requested is None:
|
||||
return reasoning_effort, None
|
||||
return requested.get("effort"), requested.get("summary")
|
||||
|
||||
|
||||
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
|
||||
|
|
@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
|
||||
"""
|
||||
|
||||
_is_base_class = False
|
||||
|
||||
temperature: int | None = None
|
||||
top_p: int | None = None
|
||||
max_tokens: int | None = None
|
||||
|
|
@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
is_magistral: Final = "magistral" in model.lower()
|
||||
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
|
||||
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
|
||||
return [
|
||||
"stream",
|
||||
"temperature",
|
||||
|
|
@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
"parallel_tool_calls",
|
||||
*(("thinking",) if is_magistral else ()),
|
||||
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
|
||||
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
|
||||
]
|
||||
|
||||
def _map_tool_choice(self, tool_choice: str) -> str:
|
||||
|
|
@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
optional_params["extra_body"] = {"random_seed": value}
|
||||
if param == "response_format":
|
||||
optional_params["response_format"] = value
|
||||
if param == "reasoning_effort" and "magistral" not in model.lower():
|
||||
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
|
||||
if (
|
||||
param == "reasoning_effort"
|
||||
and "magistral" not in model.lower()
|
||||
and (
|
||||
reasoning_level := self._reasoning_level_to_send(
|
||||
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
|
||||
)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
optional_params["reasoning_effort"] = reasoning_level
|
||||
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
|
||||
# Flag that we need to add reasoning system prompt
|
||||
optional_params["_add_reasoning_prompt"] = True
|
||||
if param == "parallel_tool_calls":
|
||||
optional_params["parallel_tool_calls"] = value
|
||||
if param in ("parallel_tool_calls", "prompt_cache_key"):
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def _reasoning_level_to_send(
|
||||
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
|
||||
) -> str | None:
|
||||
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
|
||||
sendable: Final = (
|
||||
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
|
||||
)
|
||||
if sendable is not None and summary is None:
|
||||
return sendable
|
||||
if drop_params or litellm.drop_params is True:
|
||||
return sendable
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
|
||||
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
|
||||
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
llm_provider=self.custom_llm_provider,
|
||||
model=model,
|
||||
)
|
||||
|
||||
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
|
||||
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
|
||||
hostname: Final = urlparse(resolved_api_base).hostname or ""
|
||||
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
|
||||
|
||||
@staticmethod
|
||||
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
|
||||
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
|
||||
|
||||
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
|
||||
if not self.accepts_prompt_cache_key_at(api_base):
|
||||
return "since only the Mistral API (*.mistral.ai) takes it"
|
||||
if not self.takes_prompt_cache_key_value(prompt_cache_key):
|
||||
return "since the Mistral API takes it only as a string of ASCII characters"
|
||||
return None
|
||||
|
||||
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
|
||||
requested_api_base: Final = routing.get("api_base")
|
||||
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
|
||||
allowed_openai_params: Final = routing.get("allowed_openai_params")
|
||||
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
|
||||
return False
|
||||
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
|
||||
if rejection is None:
|
||||
return False
|
||||
if litellm.drop_params is True or routing.get("drop_params") is True:
|
||||
return True
|
||||
requested_provider: Final = routing.get("custom_llm_provider")
|
||||
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
|
||||
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
|
||||
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
llm_provider=provider,
|
||||
model=model,
|
||||
)
|
||||
|
||||
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
|
||||
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
|
||||
api_base = (
|
||||
|
|
@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
messages = self._handle_message_with_file(messages)
|
||||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
file_part: Final = block.get("file") if block.get("type") == "file" else None
|
||||
file_id: Final = (
|
||||
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
|
||||
)
|
||||
if not file_id:
|
||||
return block
|
||||
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
|
||||
|
||||
@classmethod
|
||||
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return message
|
||||
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
|
||||
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
|
||||
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
|
||||
)
|
||||
|
||||
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
|
||||
"""
|
||||
Mistral API supports only 'file_id' in message content with type 'file'.
|
||||
"""
|
||||
for m in messages:
|
||||
_content_block = m.get("content")
|
||||
if _content_block and isinstance(_content_block, list):
|
||||
if any(c.get("type") == "file" for c in _content_block):
|
||||
# If file content is present, we get file_id from 'file' attribute of content block
|
||||
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
|
||||
file_contents = [c for c in _content_block if c.get("type") == "file"]
|
||||
for file_content in file_contents:
|
||||
file_id = file_content.get("file", {}).get("file_id")
|
||||
if file_id:
|
||||
# Replace 'file' with 'file_id'
|
||||
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
|
||||
file_content.pop("file", None)
|
||||
return messages
|
||||
return [self._with_mistral_file_ids(m) for m in messages]
|
||||
|
||||
def _add_reasoning_system_prompt_if_needed(
|
||||
self, messages: list[AllMessageValues], optional_params: dict
|
||||
|
|
@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
|
||||
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
|
||||
|
||||
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
|
||||
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
|
||||
withheld_params: Final = (
|
||||
("client_metadata", "prompt_cache_key")
|
||||
if "prompt_cache_key" in cache_key_params
|
||||
and self._prompt_cache_key_withheld(
|
||||
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
|
||||
)
|
||||
else ("client_metadata",)
|
||||
)
|
||||
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
|
||||
|
||||
# Call parent transform_request which handles _transform_messages
|
||||
return super().transform_request(
|
||||
|
|
|
|||
|
|
@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None",
|
||||
) -> "ModelResponse":
|
||||
"""``stream_chunk_builder`` folds every choice of a stream into one index-0
|
||||
choice, so the stream is rebuilt one choice index at a time (every chunk
|
||||
kept, its choices narrowed to that index, so usage-only chunks still
|
||||
count) and the rebuilt choices are stitched into one response, each
|
||||
carrying the index the stream gave it."""
|
||||
choice_indices: Final = tuple(
|
||||
sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
|
||||
)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue