Merge remote-tracking branch 'origin/main' into litellm_least_busy_burst_reservation

# Conflicts:
#	litellm/router.py

.circleci/scripts/run_integration.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [ "${GITHUB_ACTIONS:-}" = true ]; then
-  echo "Integration contracts are owned by CircleCI" >&2
-  exit 1
-fi
-
 suite="${1:?integration suite required}"
 mode="${2:-standard}"
 side="${3:-}"
+requested_workers="${INTEGRATION_WORKERS:-}"
+license_env=()
+if [ -n "${LITELLM_LICENSE:-}" ]; then
+  license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
+fi
 if [ "$mode" = replica ]; then
   results="test-results/integration-${suite}-replica"
 elif [ "$mode" = parity ]; then
@@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
   export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
   test -f "$LITELLM_UI_PATH/index.html"
 fi
-export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
+export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
 export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
 
 uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@@ -107,7 +107,7 @@ sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
 sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
-for service in postgres-db redis-cache; do
+for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
   address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
@@ -145,6 +145,9 @@ fi
 if [ "$suite" = mcp ]; then
   export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
 fi
+if [ -n "$requested_workers" ]; then
+  export INTEGRATION_WORKERS="$requested_workers"
+fi
 isolated_workers=false
 if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
   isolated_workers=true
@@ -185,7 +188,7 @@ start_proxy() {
     "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
-    LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+    "${license_env[@]}" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
     "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   "${shared_services[@]}" "${worker_coverage[@]}" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   LITELLM_MODE=PRODUCTION \
-  LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+  "${license_env[@]}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

.github/scripts/assert_secret_absent.py

@@ -0,0 +1,119 @@
+import base64
+import io
+import os
+import sys
+import urllib.parse
+import zipfile
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from functools import reduce
+from itertools import chain
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, TextIO
+
+SECRET_VARIABLE: Final = "SCANNED_SECRET"
+MIN_ENCODED_LENGTH: Final = 12
+MAX_ARCHIVE_DEPTH: Final = 3
+MASK: Final = "***"
+
+Forms = tuple[tuple[str, str], ...]
+
+
+@dataclass(frozen=True, slots=True)
+class Hit:
+    location: str
+    form: str
+
+
+def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
+    for offset in range(3):
+        chunk: Final = raw[offset:]
+        yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
+
+
+def secret_forms(secret: str) -> Forms:
+    encoded: Final = (
+        ("url-encoded", urllib.parse.quote(secret, safe="")),
+        *_aligned_encodings(secret.encode(), "base64", base64.b64encode),
+        *_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
+    )
+    by_value: Final = MappingProxyType(
+        {value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
+    )
+    return (("raw", secret), *((name, value) for value, name in by_value.items()))
+
+
+def mask(text: str, forms: Forms) -> str:
+    return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
+
+
+def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
+    for form, value in forms:
+        if value in name:
+            yield Hit(mask(location, forms), f"{form} in name")
+
+
+def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
+    for form, value in forms:
+        if value.encode() in data:
+            yield Hit(mask(location, forms), form)
+    if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
+        return
+    with zipfile.ZipFile(io.BytesIO(data)) as archive:
+        for member in archive.infolist():
+            if member.is_dir():
+                continue
+            yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
+
+
+def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
+    for directory, subdirectories, files in os.walk(root):
+        yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
+        yield from ((Path(directory, name), False) for name in files)
+
+
+def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
+    location: Final = str(path.relative_to(root))
+    if linked_directory:
+        yield Hit(mask(location, forms), "directory symlink, not scanned")
+        return
+    yield from _name_hits(location, location, forms)
+    yield from _content_hits(location, path.read_bytes(), forms, 0)
+
+
+def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
+    forms: Final = secret_forms(secret)
+    entries: Final = tuple(_walk(root))
+    hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
+    return len(entries), hits
+
+
+def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
+    if len(argv) != 1:
+        print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
+        return 2
+    secret: Final = environ.get(SECRET_VARIABLE, "")
+    root: Final = Path(argv[0])
+    if not secret:
+        print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
+        return 0
+    if not root.is_dir():
+        print(f"{root} is not a directory, nothing to scan", file=out)
+        return 0
+    try:
+        scanned, hits = scan(root, secret)
+    except Exception as error:
+        print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
+        return 1
+    for hit in hits:
+        print(f"secret found: {hit.location} ({hit.form})", file=out)
+    if hits:
+        print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
+        return 1
+    print(f"no copy of the secret in {scanned} files under {root}", file=out)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

.github/workflows/cost-map-guard.yml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@@ -21,16 +22,19 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head, its merge base, and the base branch tip
+      - name: Check out the base branch tip and fetch the pull request head and its merge base
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_REF: ${{ github.event.pull_request.base.ref }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
           merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
+          git checkout -q --detach "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
           echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv

.github/workflows/lens-install-smoke.yml

@@ -1,152 +0,0 @@
-name: Lens installation smoke
-
-on:
-  workflow_dispatch:
-    inputs:
-      lens_image:
-        description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      lens_upgrade_image:
-        description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      gateway_baseline_ref:
-        description: Previously qualified gateway source commit, full SHA
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: lens-install-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-images:
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-          - component: backend
-            dockerfile: backend/Dockerfile
-          - component: ui
-            dockerfile: ui/Dockerfile
-          - component: migrations
-            dockerfile: migrations/Dockerfile
-          - component: monolith
-            dockerfile: Dockerfile
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-            baseline: true
-          - component: backend
-            dockerfile: backend/Dockerfile
-            baseline: true
-          - component: monolith
-            dockerfile: Dockerfile
-            baseline: true
-    env:
-      COMPONENT: ${{ matrix.component }}
-      DOCKERFILE: ${{ matrix.dockerfile }}
-      SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
-      IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
-      CURRENT_REF: ${{ github.sha }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ env.SOURCE_REF }}
-      - name: Verify the selected source
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          test "$(git rev-parse HEAD)" = "$SOURCE_REF"
-          if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
-            test "$SOURCE_REF" != "$CURRENT_REF"
-          fi
-      - name: Build the matching release image
-        run: |
-          docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-            -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
-      - name: Save the matching release image
-        run: |
-          docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
-            | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
-          path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
-          compression-level: 0
-          retention-days: 3
-          if-no-files-found: error
-          overwrite: true
-
-  helm-install:
-    needs: build-images
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-install-*-${{ github.sha }}
-          merge-multiple: true
-          path: ${{ runner.temp }}/lens-install-images
-      - name: Load the matching release images
-        run: |
-          for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
-            gzip -dc "$archive" | docker load
-            rm "$archive"
-          done
-      - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
-        with:
-          cosign-release: v2.4.3
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ghcr.io
-          username: ${{ vars.LENS_READ_USER || github.actor }}
-          password: ${{ secrets.LENS_READ_TOKEN }}
-      - name: Verify independent Lens images before loading them
-        env:
-          LENS_IMAGE: ${{ inputs.lens_image }}
-          LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
-          LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
-          LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
-          printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
-          echo "$LENS_PUBLIC_KEY_SHA256  $RUNNER_TEMP/lens.pub" | sha256sum --check
-          for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
-            [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
-            cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
-            cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
-            docker pull "$image"
-          done
-          docker tag "$LENS_IMAGE" lens-ci-worker:baseline
-          docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
-      - name: Install pinned Kubernetes test tools
-        run: |
-          curl --fail --location --output "$RUNNER_TEMP/kind" \
-            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
-          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kind"
-          curl --fail --location --output "$RUNNER_TEMP/kubectl" \
-            https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
-          echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea  $RUNNER_TEMP/kubectl" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kubectl"
-          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
-            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
-          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
-          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
-          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
-      - name: Install, ingest, upgrade, and restart both charts
-        run: bash tests/e2e/migrations/lens_helm_smoke.sh

.github/workflows/spend-logs-guard.yml

@@ -19,17 +19,17 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          ref: ${{ github.event.pull_request.base.sha }}
+          ref: ${{ github.event.pull_request.base.ref }}
           persist-credentials: false
       - name: Fetch the pull request head, its merge base and its reviews
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

.github/workflows/test-integration.yml

@@ -0,0 +1,199 @@
+name: integration
+
+on:
+  pull_request:
+    types: [labeled]
+  schedule:
+    - cron: "5 8-20/3 * * 1-6"
+      timezone: America/Los_Angeles
+  workflow_dispatch:
+    inputs:
+      suite:
+        description: Integration group to run
+        type: choice
+        options:
+          - all
+          - providers
+          - extensions
+          - accounting
+          - management
+          - management-replica
+          - database
+          - database-replica
+          - mcp
+          - security
+          - cost
+          - sdk
+          - browser
+        default: all
+      workers:
+        description: Workers per group, blank keeps each group's default
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  integration:
+    name: integration ${{ matrix.suite }}
+    if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
+    environment: integration-license
+    runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
+    env:
+      UV_PYTHON: "3.12"
+      INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: circle_test
+        ports:
+          - 5432:5432
+        options: >-
+          --shm-size 1g
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+      redis:
+        image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Cache Prisma binaries
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Install redis-server 7.2.16
+        timeout-minutes: 3
+        run: |
+          mkdir -p "$RUNNER_TEMP/redis-bin"
+          cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
+          docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
+          docker rm "$cid"
+          "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
+          echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
+
+      - name: Install MCP SDK1 peer
+        if: matrix.suite == 'mcp'
+        timeout-minutes: 5
+        run: |
+          uv venv --python 3.12 .venv-mcp-peer
+          uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+          echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
+
+      - name: Set up Node.js
+        if: matrix.suite == 'browser'
+        timeout-minutes: 5
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version-file: ui/litellm-dashboard/.nvmrc
+          cache: "npm"
+          cache-dependency-path: |
+            ui/litellm-dashboard/package-lock.json
+            tests/e2e/ui/package-lock.json
+
+      - name: Install browser dependencies and build the dashboard
+        if: matrix.suite == 'browser'
+        timeout-minutes: 20
+        run: |
+          (cd ui/litellm-dashboard && npm ci)
+          (cd tests/e2e/ui && npm ci)
+          sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
+            node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
+          node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
+          (cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
+
+      - name: Run integration contracts
+        timeout-minutes: 45
+        env:
+          SUITE: ${{ matrix.suite }}
+          INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
+          INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
+          LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
+        run: |
+          case "$SUITE" in
+            *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
+            *) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
+          esac
+
+      - name: Verify every case ran exactly once
+        if: ${{ !cancelled() && matrix.suite != 'browser' }}
+        timeout-minutes: 2
+        run: |
+          record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
+          test -n "$record"
+          mkdir -p "$RUNNER_TEMP/shards/0"
+          cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
+          .venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
+
+      - name: Report MCP coverage
+        if: ${{ !cancelled() && matrix.suite == 'mcp' }}
+        timeout-minutes: 5
+        run: |
+          coverage_dir=test-results/integration-mcp/coverage
+          test -f "$coverage_dir/coverage.txt" && exit 0
+          export COVERAGE_FILE="$PWD/$coverage_dir/data"
+          .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
+          .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
+          .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
+          tail -n 1 "$coverage_dir/coverage.txt"
+
+      - name: Check results for the license
+        id: secret-scan
+        if: ${{ !cancelled() }}
+        timeout-minutes: 5
+        env:
+          SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
+        run: python .github/scripts/assert_secret_absent.py test-results
+
+      - name: Upload results
+        if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
+        timeout-minutes: 10
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: test-results/
+          retention-days: 14

.github/workflows/test-linting.yml

@@ -48,11 +48,13 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$MERGE_BASE"
           retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
           echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@@ -264,6 +266,7 @@ jobs:
     name: ui
     permissions:
       contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     timeout-minutes: 8
     defaults:
@@ -281,16 +284,12 @@ jobs:
         id: changed
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          # base.sha is the base branch tip from when the PR was opened, while
-          # actions/checkout leaves HEAD on a merge of the PR into the *current*
-          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
-          # landed since, so a PR that touches no UI file still gets linted
-          # against hundreds of other people's files. Diff the PR head against its
-          # own merge base instead, which is exactly what this PR changed.
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"
@@ -630,6 +629,12 @@ jobs:
       - name: check_migrations_lock_timeout
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
 
+      - name: check_no_none_string_error_fields
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
+
+      - name: test_check_no_none_string_error_fields
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
+
       - name: check_no_publicly_known_master_key
         run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
 

.github/workflows/test-unit.yml

@@ -227,6 +227,7 @@ jobs:
               tests/unit/llms/test_oss_decision.py
               tests/unit/llms/test_polling_url_origin_match.py
               tests/unit/llms/test_predibase_transformation.py
+              tests/unit/llms/test_subscription_accounts.py
               tests/unit/llms/a2a
               tests/unit/llms/aiml
               tests/unit/llms/aiohttp_openai
@@ -245,6 +246,7 @@ jobs:
               tests/unit/llms/chatgpt
               tests/unit/llms/clarifai
               tests/unit/llms/claude_code
+              tests/unit/llms/clf_ai_gateway
               tests/unit/llms/cloudflare
               tests/unit/llms/codestral
               tests/unit/llms/codex
@@ -1003,9 +1005,9 @@ jobs:
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1150,18 +1152,20 @@ jobs:
         env:
           CI: "true"
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
 
-          if [ -z "$BASE_SHA" ]; then
+          if [ -z "$PR_NUMBER" ]; then
             echo "Push to $GITHUB_REF_NAME: running the full suite"
             full_suite
             exit 0
           fi
 
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           changed_files=()
@@ -1451,9 +1455,9 @@ jobs:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
         run: |
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
           fi

Makefile

@@ -153,7 +153,7 @@ lint-install:
 # the same set CI's ':(glob)litellm/**/*.py' selects.
 lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	@base_ref=$$($(RESOLVE_BASE)) && \
-	changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
+	changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
 	files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
 	if [ -z "$$files" ]; then \
 		echo "No changed litellm Python files to format-check."; \

README.md

@@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -398,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) |  |  |  |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/routes/allowlist.py

@@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/subscription_accounts/",
     "/cost_optimization/",
     "/cost/",
     "/credentials",

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.80"
+version = "0.1.81"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.80"
+version = "0.1.81"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
         "/test",
         "/debug/memory/summary",
         "/api/event_logging/batch",
+        "/eu_jev/v1/systemone",
     }
 )
 

helm/litellm-helm/Chart.lock

@@ -5,8 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
-generated: '2026-10-09T04:18:10.643228000Z'
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: '2026-10-30T02:13:11.505726000Z'

helm/litellm-helm/Chart.yaml

@@ -39,6 +39,3 @@ dependencies:
     version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm-helm/README.md

@@ -256,9 +256,7 @@ itself.
 
 ## Connect Lens
 
-`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 ## Shutdown accounting
 

helm/litellm-helm/templates/_helpers.tpl

@@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
   value: {{ .Values.collector.drainTimeoutSeconds | quote }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -402,95 +398,28 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.labels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
-{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm-helm/templates/ingress.yaml

@@ -1,8 +1,6 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "litellm.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- $lensMode := include "litellm.lens.mode" . -}}
-{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@@ -46,18 +44,18 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
+          {{- if and $.Values.lens.enabled $.Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
-                name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
+                name: {{ $.Values.lens.service.name }}
                 port:
-                  number: {{ $.Values.lensWorker.service.port }}
+                  number: {{ $.Values.lens.service.port }}
               {{- else }}
-              serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
-              servicePort: {{ $.Values.lensWorker.service.port }}
+              serviceName: {{ $.Values.lens.service.name }}
+              servicePort: {{ $.Values.lens.service.port }}
               {{- end }}
           {{- end }}
           {{- range .paths }}

helm/litellm-helm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm-helm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm-helm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm-helm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm-helm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm-helm/tests/lens_connection_tests.yaml

@@ -0,0 +1,128 @@
+suite: lens connection
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - ingress.yaml
+tests:
+  - it: connects the proxy to an external Lens
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service ahead of the host paths
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[0].path
+          value: /lens-ingest
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,84 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - deployment.yaml
-  - configmap-litellm.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest

helm/litellm-helm/tests/lens_modes_tests.yaml

@@ -1,187 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- ingress.yaml
-- configmap-litellm.yaml
-- deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm-helm/tests/lens_modes_validation_tests.yaml

@@ -1,52 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm-helm/tests/lens_saved_secrets_tests.yaml

@@ -1,59 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

helm/litellm-helm/tests/lens_service_tests.yaml

@@ -1,167 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects deployment.yaml to the shared Lens service
-  template: deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.hosts:
-    - host: gateway.example
-      paths:
-      - path: /
-        pathType: Prefix
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in deployment.yaml
-  template: deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference

helm/litellm-helm/tests/lens_setup_tests.yaml

@@ -1,162 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60

helm/litellm-helm/values.yaml

@@ -668,70 +668,15 @@ serviceMonitor:
     # - test-namespace
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

helm/litellm/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
-generated: '2026-10-09T04:18:10.642706000Z'

helm/litellm/Chart.yaml

@@ -6,8 +6,3 @@ version: 0.1.0
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
-
-dependencies:
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm/README.md

@@ -1,10 +1,8 @@
 # LiteLLM componentized chart
 
-This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
+This chart deploys separate gateway, backend and UI services
 
-Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
 

helm/litellm/templates/_helpers.tpl

@@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -635,93 +631,27 @@ shutdown drain window.
 {{- end -}}
 
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -166,14 +166,14 @@ spec:
     - host: {{ $componentHosts.gateway | quote }}
       http:
         paths:
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           - path: /
             pathType: Prefix
@@ -249,15 +249,15 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           {{- /*
             --- Operator-supplied extra paths (ingress.extraPaths) ---

helm/litellm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm/tests/component_hosts_tests.yaml

@@ -15,7 +15,6 @@ tests:
       ingress.componentHosts.ui: ui.example.com
       ingress.componentHosts.backend: backend.example.com
       ingress.componentHosts.gateway: gateway.example.com
-      lensWorker.mode: disabled
       ingress.tls:
         - hosts:
             - ui.example.com
@@ -83,12 +82,12 @@ tests:
       ui.backendUrl: https://backend.example.com
       ui.publicUrl: https://ui.example.com
       backend.gatewayUrl: https://gateway.example.com
-      lensWorker.mode: external
-      lensWorker.externalServiceName: existing-lens
-      lensWorker.externalUrl: https://lens.example.com
-      lensWorker.publicUrl: https://lens.example.com
-      lensWorker.serviceTokenSecret.name: existing-lens-signing
-      lensWorker.gateway.secretName: existing-lens-signing
+      lens.enabled: true
+      lens.service.name: existing-lens
+      lens.url: http://existing-lens:4318
+      lens.publicUrl: https://gateway.example.com/lens-ingest
+      lens.serviceTokenSecret.name: existing-lens-signing
+      lens.gatewaySecret.name: existing-lens-signing
     asserts:
       - equal:
           path: spec.rules[2].http.paths[0].path

helm/litellm/tests/lens_connection_tests.yaml

@@ -0,0 +1,166 @@
+suite: lens connection
+templates:
+  - gateway/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: connects the gateway to an external Lens
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: connects the backend to an external Lens
+    template: backend/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service on the single ingress host
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,86 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - gateway/deployment.yaml
-  - gateway/configmap.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.host: gateway.example
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_modes_tests.yaml

@@ -1,208 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-- ingress.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: keeps the ingestion endpoint on an independently deployed Lens service
-  template: ingress.yaml
-  set:
-    ingress.enabled: true
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: existing-lens
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm/tests/lens_modes_validation_tests.yaml

@@ -1,54 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm/tests/lens_saved_secrets_tests.yaml

@@ -1,61 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_service_tests.yaml

@@ -1,196 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- gateway/configmap.yaml
-- gateway/deployment.yaml
-- backend/deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects gateway/deployment.yaml to the shared Lens service
-  template: gateway/deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: connects backend/deployment.yaml to the shared Lens service
-  template: backend/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.host: gateway.example
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: gateway/deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in gateway/deployment.yaml
-  template: gateway/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: omits Lens connection settings when disabled in backend/deployment.yaml
-  template: backend/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: gateway/deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference
-values:
-- ./values/required.yaml

helm/litellm/tests/lens_setup_tests.yaml

@@ -1,164 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_worker_tests.yaml

@@ -1,79 +0,0 @@
-suite: Independent Lens release and credentials
-templates:
-  - lens/deployment.yaml
-values:
-  - ./values/required.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-  lensWorker.serviceTokenSecret.name: lens-service
-  lensWorker.clickhouseSecret.name: lens-storage
-tests:
-  - it: selects a Lens release independently of the gateway
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-      lensWorker.image.tag: 2.3.4
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:2.3.4
-  - it: retains the pinned Lens chart default when only the gateway changes
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:0.1.0-dev.0
-  - it: selects development publications explicitly
-    set:
-      lensWorker.image.repository: ghcr.io/berriai/lens-dev
-      lensWorker.image.tag: sha-0123456789abcdef
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
-  - it: preserves a verified registry mirror and digest
-    set:
-      lensWorker.image.repository: registry.example/lens
-      lensWorker.image.tag: replaced-label
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: rejects a malformed digest
-    set:
-      lensWorker.image.digest: sha256:invalid
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
-  - it: keeps the legacy enablement switch working
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments: {count: 0}
-  - it: keeps isolated local analysis with delegated gateway access
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content: {name: LENS_MODE, value: standalone}
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_GATEWAY_SECRET
-            valueFrom:
-              secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
-      - equal:
-          path: spec.template.spec.automountServiceAccountToken
-          value: false
-      - equal:
-          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
-          value: true
-      - equal:
-          path: spec.template.spec.volumes[0].emptyDir
-          value: {medium: Memory, sizeLimit: 1Gi}

helm/litellm/values.yaml

@@ -654,70 +654,15 @@ ui:
   topologySpreadConstraints: []
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql

@@ -0,0 +1,21 @@
+SET LOCAL lock_timeout = '10s';
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
+    "subscription_account_id" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "account_id" TEXT NOT NULL,
+    "label" TEXT,
+    "monthly_fee" DOUBLE PRECISION NOT NULL,
+    "currency" TEXT NOT NULL,
+    "billing_period_start" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT NOT NULL,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.111"
+version = "0.4.112"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.111"
+version = "0.4.112"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -238,7 +238,8 @@ impl AnthropicBeta {
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
-            | Self::ThinkingBindingControls20260801 => &[],
+            | Self::ThinkingBindingControls20260801
+            | Self::ThinkingDisplayUpdates20260818 => &[],
             Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
@@ -275,7 +276,6 @@ impl AnthropicBeta {
                 BetaProvider::Databricks,
             ],
             Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
             Self::ToolSearchTool20251019 => &[
                 BetaProvider::Anthropic,
                 BetaProvider::AzureAi,

litellm/__init__.py

@@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
 baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
+clf_ai_gateway_models: set[str] = set()  # mutable-ok: filled at import like sibling sets
 perplexity_models: Set = set()
 watsonx_models: Set = set()
 gemini_models: Set = set()
@@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             bedrock_converse_models.add(key)
         elif value.get("litellm_provider") == "deepinfra":
             deepinfra_models.add(key)
+        elif value.get("litellm_provider") == "clf_ai_gateway":  # pyright: ignore[reportUnknownMemberType]  # raw cost map
+            clf_ai_gateway_models.add(key)  # pyright: ignore[reportUnknownArgumentType]  # raw cost map key
         elif value.get("litellm_provider") == "perplexity":
             perplexity_models.add(key)
         elif value.get("litellm_provider") == "watsonx":
@@ -869,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key and "fireworks-ai-default" not in key:
                 fireworks_ai_models.add(key)
+        elif value.get("litellm_provider") == "together_ai":
+            if _is_past_deprecation_date(value):
+                together_ai_models.discard(key)
+            elif not key.startswith("together-ai-"):
+                together_ai_models.add(key)
         elif value.get("litellm_provider") == "baseten":
             if _is_past_deprecation_date(value):
                 baseten_models.discard(key)
@@ -1110,6 +1118,7 @@ model_list = list(
     | set(ollama_models)
     | bedrock_models
     | deepinfra_models
+    | clf_ai_gateway_models
     | perplexity_models
     | set(maritalk_models)
     | runwayml_models
@@ -1235,6 +1244,7 @@ def _build_models_by_provider() -> dict:
         "ollama": ollama_models,
         "ollama_chat": ollama_models,
         "deepinfra": deepinfra_models,
+        "clf_ai_gateway": clf_ai_gateway_models,
         "perplexity": perplexity_models,
         "maritalk": maritalk_models,
         "watsonx": watsonx_models,
@@ -1783,6 +1793,9 @@ if TYPE_CHECKING:
     from .llms.bespoke.decisions.transformation import (
         BespokeDecisionsConfig as BespokeDecisionsConfig,
     )
+    from .llms.eu_jev.decisions.transformation import (
+        EuJevDecisionsConfig as EuJevDecisionsConfig,
+    )
     from .llms.openai.decisions.transformation import (
         OpenAIDecisionsConfig as OpenAIDecisionsConfig,
     )
@@ -2108,6 +2121,9 @@ if TYPE_CHECKING:
         LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
     )
     from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
+    from .llms.clf_ai_gateway.chat.transformation import (
+        ClfAiGatewayConfig as _ClfAiGatewayConfig,
+    )
     from .llms.llamafile.chat.transformation import (
         LlamafileChatConfig as _LlamafileChatConfig,
     )
@@ -2137,6 +2153,7 @@ if TYPE_CHECKING:
     IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
     LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
     DeepInfraConfig: Type[_DeepInfraConfig]
+    ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
     LlamafileChatConfig: Type[_LlamafileChatConfig]
     LMStudioChatConfig: Type[_LMStudioChatConfig]
     LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]

litellm/_lazy_imports_registry.py

@@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
     "OllamaDecisionsConfig",
     "LayaDecisionsConfig",
     "BespokeDecisionsConfig",
+    "EuJevDecisionsConfig",
     "OpenAIDecisionsConfig",
     "AzureDecisionsConfig",
     "AzureAIDecisionsConfig",
@@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
     "MistralEmbeddingConfig",
     "OpenAIImageVariationConfig",
     "DeepInfraConfig",
+    "ClfAiGatewayConfig",
     "DeepgramAudioTranscriptionConfig",
     "TopazImageVariationConfig",
     "OpenAITextCompletionConfig",
@@ -753,6 +755,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
     "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
     "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
+    "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
     "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
     "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
@@ -973,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "OpenAIImageVariationConfig",
     ),
     "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
+    "ClfAiGatewayConfig": (
+        ".llms.clf_ai_gateway.chat.transformation",
+        "ClfAiGatewayConfig",
+    ),
     "DeepgramAudioTranscriptionConfig": (
         ".llms.deepgram.audio_transcription.transformation",
         "DeepgramAudioTranscriptionConfig",

litellm/_logging.py

@@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
 from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.secret_redaction import (
+    REDACTED,
     python_redact_string,
     python_redact_structured_value,
     redact_internal_details,
+    redact_structured_value,
 )
 from litellm.litellm_core_utils.secret_redaction import (
     redact_string as redact_secret_string,
@@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
     return redact_internal_details(value)
 
 
+_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
+_CLIENT_ITEMS: Final = TypeAdapter(list[object])
+
+
+def _redacted_client_field(key: str | None, value: object) -> object:
+    if isinstance(value, str):
+        return redact_internal_details(redact_structured_value(key, value))
+    if isinstance(value, dict):
+        return {
+            child_key: _redacted_client_field(child_key, child)
+            for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
+        }
+    if isinstance(value, (list, tuple)):
+        return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
+    if not isinstance(value, (int, float)):
+        return value
+    text: Final = str(value)
+    return value if redact_structured_value(key, text) == text else REDACTED
+
+
+def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
+    if not _ENABLE_SECRET_REDACTION:
+        return fields
+    return _redacted_client_field(None, dict(fields))
+
+
 def _substituted_color_message(record: logging.LogRecord) -> str | None:
     """Render a record's ``color_message`` against its args, or None if absent.
 

litellm/caching/Readme.md

@@ -8,10 +8,9 @@ The following caching mechanisms are supported:
 2. **RedisSemanticCache**
 3. **QdrantSemanticCache**
 4. **InMemoryCache**
-5. **DiskCache**
-6. **S3Cache**
-7. **AzureBlobCache**
-8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
+5. **S3Cache**
+6. **AzureBlobCache**
+7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
 
 ## Folder Structure
 
@@ -20,7 +19,6 @@ litellm/caching/
 ├── base_cache.py
 ├── caching.py
 ├── caching_handler.py
-├── disk_cache.py
 ├── dual_cache.py
 ├── in_memory_cache.py
 ├── qdrant_semantic_cache.py

litellm/caching/__init__.py

@@ -1,6 +1,5 @@
 from .azure_blob_cache import AzureBlobCache
 from .caching import Cache, LiteLLMCacheType
-from .disk_cache import DiskCache
 from .dual_cache import DualCache
 from .gcs_cache import GCSCache
 from .in_memory_cache import InMemoryCache

litellm/caching/caching.py

@@ -18,7 +18,7 @@ from contextlib import contextmanager
 from enum import Enum
 from typing import Any, Final, Literal, cast
 
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 
 import litellm
 from litellm._internal_context import current_service_target, service_target
@@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 
 from .azure_blob_cache import AzureBlobCache
 from .base_cache import BaseCache
-from .disk_cache import DiskCache
 from .dual_cache import DualCache  # noqa: F401  # re-exported, callers import DualCache from litellm.caching.caching
 from .gcs_cache import GCSCache
 from .in_memory_cache import InMemoryCache
@@ -44,6 +43,9 @@ from .s3_cache import S3Cache
 RESPONSE_CACHE_TARGET: Final = "llm_response"
 _ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
 _DECODED_LITERAL: Final = TypeAdapter(object)
+_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
+    list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
 
 
 @contextmanager
@@ -153,7 +155,7 @@ class Cache:
         Initializes the cache based on the given type.
 
         Args:
-            type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
+            type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
 
             # Redis Cache Args
             host (str, optional): The host address for the Redis cache. Required if type is "redis".
@@ -174,9 +176,6 @@ class Cache:
             semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
             max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
 
-            # Disk Cache Args
-            disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
-
             # S3 Cache Args
             s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
             s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
@@ -211,7 +210,7 @@ class Cache:
             if not redis_startup_nodes:
                 _env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
                 if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
-                    redis_startup_nodes = json.loads(_env_cluster_nodes)
+                    redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
 
             if redis_startup_nodes:
                 # Only pass GCP parameters if they are provided
@@ -306,8 +305,11 @@ class Cache:
                 account_url=azure_account_url,
                 container=azure_blob_container,
             )
-        elif type == LiteLLMCacheType.DISK:
-            self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
+        else:
+            supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
+            raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
+        if disk_cache_dir is not None:
+            verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
         if "cache" not in litellm.input_callback:
             litellm.input_callback.append("cache")
         if "cache" not in litellm.success_callback:
@@ -1044,7 +1046,7 @@ def enable_cache(
     Enable cache with the specified configuration.
 
     Args:
-        type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
+        type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
         host (Optional[str]): The host address of the cache server. Defaults to None.
         port (Optional[str]): The port number of the cache server. Defaults to None.
         password (Optional[str]): The password for the cache server. Defaults to None.
@@ -1056,7 +1058,7 @@ def enable_cache(
         None
 
     Raises:
-        None
+        ValueError: If no cache is set yet and the cache type is unsupported.
     """
     print_verbose("LiteLLM: Enabling Cache")
     if "cache" not in litellm.input_callback:
@@ -1091,7 +1093,7 @@ def update_cache(
     Update the cache for LiteLLM.
 
     Args:
-        type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
+        type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
         host (Optional[str]): The host of the cache. Defaults to None.
         port (Optional[str]): The port of the cache. Defaults to None.
         password (Optional[str]): The password for the cache. Defaults to None.

litellm/caching/caching_handler.py

@@ -9,7 +9,7 @@ This file is a wrapper around caching.py
 
 This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
 
-It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
+It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
 
 In each method it will call the appropriate method from caching.py
 """
@@ -830,6 +830,13 @@ class LLMCachingHandler:
         if is_internal_call.get():
             return
 
+        if logging_obj.defer_async_logging and not logging_obj.stream:
+            from litellm.utils import dispatch_success_logging
+
+            logging_obj.model_call_details["cache_hit"] = cache_hit  # rebind-ok: capture cache status before deferring
+            dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
+            return
+
         from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 
         GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

litellm/caching/disk_cache.py

@@ -1,92 +0,0 @@
-import json
-from typing import TYPE_CHECKING, Any, Final
-
-from pydantic import TypeAdapter
-
-from .base_cache import BaseCache
-
-if TYPE_CHECKING:
-    from opentelemetry.trace import Span as _Span
-
-    Span = _Span
-else:
-    Span = Any
-
-_DECODED_DISK_VALUE: Final = TypeAdapter(object)
-
-
-class DiskCache(BaseCache):
-    def __init__(self, disk_cache_dir: str | None = None):
-        try:
-            import diskcache as dc
-        except ModuleNotFoundError as e:
-            raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
-
-        # if users don't provider one, use the default litellm cache
-        if disk_cache_dir is None:
-            self.disk_cache = dc.Cache(".litellm_cache")
-        else:
-            self.disk_cache = dc.Cache(disk_cache_dir)
-
-    def set_cache(self, key, value, **kwargs):
-        if "ttl" in kwargs:
-            self.disk_cache.set(key, value, expire=kwargs["ttl"])
-        else:
-            self.disk_cache.set(key, value)
-
-    async def async_set_cache(self, key, value, **kwargs):
-        self.set_cache(key=key, value=value, **kwargs)
-
-    async def async_set_cache_pipeline(self, cache_list, **kwargs):
-        for cache_key, cache_value in cache_list:
-            if "ttl" in kwargs:
-                self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
-            else:
-                self.set_cache(key=cache_key, value=cache_value)
-
-    def get_cache(self, key, **kwargs):
-        original_cached_response: Final = self.disk_cache.get(key)
-        if original_cached_response:
-            try:
-                cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
-            except Exception:
-                cached_response = original_cached_response
-            return cached_response
-        return None
-
-    def batch_get_cache(self, keys: list, **kwargs):
-        return_val: Final = []
-        for k in keys:
-            val = self.get_cache(key=k, **kwargs)
-            return_val.append(val)
-        return return_val
-
-    def increment_cache(self, key, value: int, **kwargs) -> int:
-        with self.disk_cache.transact():
-            cached_value: Final = self.get_cache(key=key)
-            init_value: Final = cached_value if isinstance(cached_value, int) else 0
-            new_value: Final = init_value + value
-            self.set_cache(key, new_value, **kwargs)
-            return new_value
-
-    async def async_get_cache(self, key, **kwargs):
-        return self.get_cache(key=key, **kwargs)
-
-    async def async_batch_get_cache(self, keys: list, **kwargs):
-        return_val: Final = []
-        for k in keys:
-            val = self.get_cache(key=k, **kwargs)
-            return_val.append(val)
-        return return_val
-
-    async def async_increment(self, key, value: int, **kwargs) -> int:
-        return self.increment_cache(key=key, value=value, **kwargs)
-
-    def flush_cache(self):
-        self.disk_cache.clear()
-
-    async def disconnect(self):
-        pass
-
-    def delete_cache(self, key):
-        self.disk_cache.pop(key)

litellm/caching/evicted_client_closer.py

@@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
 close them, and each bucket is ordered by deadline, so a reap walks the entries
 that are due rather than the whole queue.
 
+A bucket whose loop has been collected is dropped, and its clients are left to the
+collector, as they are when the queue is full.
+
 The queue holds its clients weakly, so waiting out a grace window never keeps
 alive anything the collector would have reclaimed first.
 """
@@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
 import asyncio
 import contextlib
 import inspect
+import itertools
 import threading
 import time
 import weakref
@@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
 _CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
 
 _BucketKey = str | int
+_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
+_LOOP_TOKEN_COUNTER: Final = itertools.count()
+_LOOP_TOKEN_LOCK: Final = threading.Lock()
 
 
 @dataclass(frozen=True, slots=True)
@@ -65,12 +72,12 @@ class _PendingClose:
     collector would otherwise have reclaimed first.
 
     ``needs_loop`` is set for a client whose close is a coroutine; those can only
-    be closed from the event loop they were evicted on, recorded in ``loop_id``.
+    be closed from the event loop they were evicted on, recorded in ``loop_token``.
     A client that closes synchronously carries neither constraint.
     """
 
     client_ref: "weakref.ref[object]"
-    loop_id: int | None
+    loop_token: int | None
     needs_loop: bool
     close_after: float
 
@@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
     """Which reaps can close this entry: any at all, any running a loop, or one loop's."""
     if not pending.needs_loop:
         return _CLOSABLE_ANYWHERE
-    if pending.loop_id is None:
+    if pending.loop_token is None:
         return _CLOSABLE_ON_ANY_LOOP
-    return pending.loop_id
+    return pending.loop_token
 
 
-def _running_loop_id() -> int | None:
+def running_loop_token() -> int | None:
     try:
-        return id(asyncio.get_running_loop())
+        loop: Final = asyncio.get_running_loop()
     except RuntimeError:
         return None
+    try:
+        token: Final[int | None] = _LOOP_TOKENS.get(loop)
+        if token is not None:
+            return token
+        with _LOOP_TOKEN_LOCK:
+            return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
+    except TypeError:
+        return id(loop)
 
 
 def _close_function(client: object) -> Callable[[], object] | None:
@@ -186,6 +201,8 @@ class EvictedClientCloser:
         self._clock = clock
         self._owned: weakref.WeakSet[object] = weakref.WeakSet()
         self._buckets: dict[_BucketKey, deque[_PendingClose]] = {}  # mutable-ok: deadline-ordered queues
+        self._dead_loop_tokens: deque[int] = deque()  # mutable-ok: loop finalizers append without taking the queue lock
+        self._watched_loop_tokens: set[int] = set()  # mutable-ok: tracks tokens until their dead buckets are discarded
         self._pending_count = 0
         self._queue_lock = threading.Lock()  # the cache is reachable from every worker thread's loop
         self._close_tasks: set[asyncio.Task[None]] = set()  # mutable-ok: strong refs to running closes
@@ -216,13 +233,19 @@ class EvictedClientCloser:
         close_fn: Final = _close_function(client)
         if close_fn is None:
             return
+        loop_token: Final = running_loop_token()
+        needs_loop: Final = inspect.iscoroutinefunction(close_fn)
         if self._pending_count >= self._max_pending:
-            return
+            self._discard_dead_loops()
+            if self._pending_count >= self._max_pending:
+                return
+        if needs_loop and loop_token is not None:
+            self._watch_loop(loop_token)
         self._enqueue(
             _PendingClose(
                 client_ref=weakref.ref(client),
-                loop_id=_running_loop_id(),
-                needs_loop=inspect.iscoroutinefunction(close_fn),
+                loop_token=loop_token,
+                needs_loop=needs_loop,
                 close_after=self._clock() + self._grace_seconds,
             )
         )
@@ -236,7 +259,7 @@ class EvictedClientCloser:
         if not self._pending_count:
             return
         now: Final = self._clock()
-        for pending in self._take_due(_running_loop_id(), now):
+        for pending in self._take_due(running_loop_token(), now):
             client = pending.client_ref()
             if client is None:
                 continue
@@ -249,6 +272,29 @@ class EvictedClientCloser:
     def pending_count(self) -> int:
         return self._pending_count
 
+    def _watch_loop(self, loop_token: int) -> None:
+        if loop_token in self._watched_loop_tokens:
+            return
+        try:
+            weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
+        except TypeError:
+            return
+        self._watched_loop_tokens.add(loop_token)
+
+    def _discard_dead_loops(self) -> None:
+        with self._queue_lock:
+            self._discard_dead_loops_locked()
+
+    def _discard_dead_loops_locked(self) -> None:
+        while self._dead_loop_tokens:
+            self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
+
+    def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
+        self._watched_loop_tokens.discard(loop_token)
+        bucket: Final = self._buckets.pop(loop_token, None)
+        if bucket is not None:
+            self._pending_count -= len(bucket)
+
     def _enqueue(self, pending: _PendingClose) -> None:
         """Append to the entry's bucket, dropping any dead entries it queues behind.
 
@@ -257,6 +303,7 @@ class EvictedClientCloser:
         the front rather than having to be searched for.
         """
         with self._queue_lock:
+            self._discard_dead_loops_locked()
             bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
             while bucket and bucket[0].client_ref() is None:
                 bucket.popleft()
@@ -264,9 +311,12 @@ class EvictedClientCloser:
             bucket.append(pending)
             self._pending_count += 1
 
-    def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
-        buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
+    def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
+        buckets = (
+            (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
+        )
         with self._queue_lock:
+            self._discard_dead_loops_locked()
             return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
 
     def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

litellm/caching/llm_caching_handler.py

@@ -2,10 +2,9 @@
 Add the event loop to the cache key, to prevent event loop closed errors.
 """
 
-import asyncio
 from typing import Final
 
-from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
+from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
 from .in_memory_cache import InMemoryCache
 
 
@@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
         self.evicted_client_closer.schedule(evicted)
         self.evicted_client_closer.reap()
 
-    def update_cache_key_with_event_loop(self, key):
+    def update_cache_key_with_event_loop(self, key: str) -> str:
         """
         Add the event loop to the cache key, to prevent event loop closed errors.
         If none, use the key as is.
         """
-        try:
-            event_loop: Final = asyncio.get_running_loop()
-            stringified_event_loop: Final = str(id(event_loop))
-            return f"{key}-{stringified_event_loop}"
-        except RuntimeError:  # handle no current running event loop
-            return key
+        token: Final = running_loop_token()
+        return f"{key}-{token}" if token is not None else key
 
-    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+    def _scoped_key(self, key: str, loop_bound: bool) -> str:
+        return self.update_cache_key_with_event_loop(key) if loop_bound else key
+
+    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
         """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
         if litellm_owned_client:
             self.evicted_client_closer.mark_owned(value)
-        key = self.update_cache_key_with_event_loop(key)
-        return super().set_cache(key, value, **kwargs)
+        return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
 
     async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
         if litellm_owned_client:
@@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
         key = self.update_cache_key_with_event_loop(key)
         return await super().async_set_cache(key, value, **kwargs)
 
-    def get_cache(self, key, **kwargs):
-        key = self.update_cache_key_with_event_loop(key)
+    def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
+        scoped_key: Final = self._scoped_key(key, loop_bound)
         self.evicted_client_closer.reap()
 
-        return super().get_cache(key, **kwargs)
+        return super().get_cache(scoped_key, **kwargs)
 
-    async def async_get_cache(self, key, **kwargs):
+    async def async_get_cache(self, key: str, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
 
         return await super().async_get_cache(key, **kwargs)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
         self._structured_output: Final = structured_output
         self._first_message_item_done: bool = False
+        self._reasoning_summary_streamed: bool = False
 
     def _handle_string_chunk(
         self, str_line: Union[str, "BaseModel"]
@@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         tool_call_index_map: dict[int, int] | None = None,  # mutable-ok: per-stream state, remapped in place
         replays_openai_history: bool = False,
         structured_output: bool = False,
+        reasoning_summary_streamed: bool = False,
     ) -> "ModelResponseStream":
         """
         Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             tool_call_index_map: Per-stream output_index -> sequential tool_call index map
             replays_openai_history: Record the completed output's item sequence for the next turn
             structured_output: Only the first message item became content
+            reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
 
         Returns:
             ModelResponseStream: OpenAI-formatted streaming chunk
@@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 )
             else:
                 raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
+        elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
+            return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
         elif event_type == "response.reasoning_summary_text.delta":
             content_part = parsed_chunk.get("delta", None)
             if content_part:
                 return ModelResponseStream(
                     choices=[
                         StreamingChoices(
-                            index=cast(int, parsed_chunk.get("summary_index")),
+                            index=0,
                             delta=Delta(reasoning_content=content_part),
                         )
                     ]
@@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 tool_call_index_map=self._tool_call_index_map,
                 replays_openai_history=self._replays_openai_history,
                 structured_output=self._structured_output,
+                reasoning_summary_streamed=self._reasoning_summary_streamed,
             )
         )
         if self._structured_output and _is_message_item_done(chunk):
             self._first_message_item_done = True
+        if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
+            self._reasoning_summary_streamed = True
         return self._with_served_service_tier(self._with_stream_scoped_id(translated))
 
     def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

litellm/constants.py

@@ -7,6 +7,7 @@ from typing import Final, Literal
 
 from litellm.litellm_core_utils.env_utils import (
     get_env_float,
+    get_env_float_in_range,
     get_env_float_or_none,
     get_env_int,
     get_env_int_in_range,
@@ -67,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
         "retry_after",
         "fallbacks",
         "context_window_fallbacks",
+        "content_policy_fallbacks",
         "retry_policy",
         "model_group_retry_policy",
         "model_group_alias",
@@ -149,6 +151,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
     "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
 )  # default cooldown a deployment if 50% of requests fail in a given minute
 DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
+NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
+    "NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
+)
 DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
 DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
 DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
@@ -416,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
     "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
 )
 BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
+REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
+REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
 
 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
 WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@@ -753,7 +760,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
 ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
+OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
+OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
+PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
+PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
+OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
+OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
 VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
 ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
     "low": 1,
@@ -813,6 +826,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
     "ollama",
     "ollama_chat",
     "deepinfra",
+    "clf_ai_gateway",
     "perplexity",
     "mistral",
     "groq",
@@ -886,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
     }
 )
 
+SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
 A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
@@ -1012,6 +1027,7 @@ openai_compatible_endpoints: Final[list[str]] = [
     "api.perplexity.ai",
     "api.endpoints.anyscale.com/v1",
     "api.deepinfra.com/v1/openai",
+    "api.clfaigateway.dev/v1",
     "api.mistral.ai/v1",
     "codestral.mistral.ai/v1/chat/completions",
     "codestral.mistral.ai/v1/fim/completions",
@@ -1083,6 +1099,7 @@ openai_compatible_providers: Final[list[str]] = [
     "deepseek",
     "tencent",
     "deepinfra",
+    "clf_ai_gateway",
     "perplexity",
     "xinference",
     "xai",
@@ -1261,7 +1278,7 @@ empower_models: Final = set(
     ]
 )
 
-together_ai_models: Final[set] = set(
+together_ai_models: Final[set[str]] = set(
     [
         # llama llms - chat
         "togethercomputer/llama-2-70b-chat",

litellm/cost_calculator.py

@@ -153,7 +153,9 @@ from litellm.utils import (
     ProviderConfigManager,
     TextCompletionResponse,
     TranscriptionResponse,
+    cached_cost_map_key,
     cached_get_model_info_helper,
+    mapped_model_fill_missing,
     model_info_with_pricing_overrides,
     runtime_model_cost_registrations,
     token_counter,
@@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
     """
     1. If custom pricing is true, return received model name
     2. If base_model is set (e.g. for azure models), return that
-    3. If completion response has model set return that
+    3. If completion response has model set return that, unless only the requested model's cost-map row has rates
     4. Check if model is passed in return that
     """
 
     return_model: str | None = None
     custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
 
-    completion_response_model: str | None = None
-    if completion_response is not None:
-        if isinstance(completion_response, BaseModel):
-            completion_response_model = getattr(completion_response, "model", None)
-        elif isinstance(completion_response, dict):
-            completion_response_model = completion_response.get("model", None)
+    completion_response_model: Final = _get_response_model(completion_response)
     hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
+    requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
     configured_model: Final = model or (
         completion_response_model if isinstance(completion_response_model, str) else None
@@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
         else:
             return_model = model
 
-    elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
-        return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
+    elif base_model is not None or bedrock_pricing_model is not None:
+        return_model = base_model if base_model is not None else bedrock_pricing_model
+
+    elif provider_response_model is not None:
+        return _served_or_priced_requested_model(
+            provider_response_model, requested_model, custom_llm_provider, priced_region
+        )
 
     elif completion_response_model is None and hidden_params is not None:
         if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
             return_model = hidden_params.get("model", model)
 
     if return_model is None and completion_response_model is not None:
-        return_model = completion_response_model
+        return _served_or_priced_requested_model(
+            completion_response_model, requested_model, custom_llm_provider, priced_region
+        )
 
     if return_model is None and model is not None:
         return_model = model
 
-    if (
-        return_model is not None
-        and custom_llm_provider is not None
-        and not _model_contains_known_llm_provider(return_model)
-    ):  # add provider prefix if not already present, to match model_cost
-        provider_prefix: Final = (
-            custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
-        )
-        return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
-
-    return return_model
+    if return_model is None:
+        return None
+    return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
 
 
 _select_model_name_for_cost_calc = select_model_name_for_cost_calc
 
 
+def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
+    if custom_llm_provider is None or _model_contains_known_llm_provider(model):
+        return model
+    provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
+    return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
+
+
+def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
+    rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
+    return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
+
+
+def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
+    key: Final = cached_cost_map_key(model, custom_llm_provider)
+    if key is None:
+        return False
+    entry: Final = _raw_cost_map_entry(key)
+    if entry is None:
+        return _rule_declares_rates(model, custom_llm_provider)
+    fill_missing: Final = mapped_model_fill_missing(key, entry)
+    return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
+
+
+def _served_or_priced_requested_model(
+    served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
+) -> str:
+    served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
+    if requested_model is None:
+        return served
+    requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
+    if served in (requested, requested.partition("/")[2]):
+        return served
+    pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
+    if _resolves_to_rates(served, pricing_provider):
+        return served
+    return requested if _resolves_to_rates(requested, pricing_provider) else served
+
+
 def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
     """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
     registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@@ -1509,6 +1544,12 @@ def _response_time_ms_for_cost(
     return litellm_logging_obj.get_response_ms()
 
 
+def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
+    breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
+    total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
+    return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
+
+
 def _adjusted_provider_billed_cost(
     reported_cost: float,
     custom_llm_provider: str | None,
@@ -1526,9 +1567,9 @@ def _adjusted_provider_billed_cost(
     )
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=None,
-        completion_tokens_cost_usd_dollar=None,
-        cost_for_built_in_tools_cost_usd_dollar=None,
+        prompt_tokens_cost_usd_dollar=0.0,
+        completion_tokens_cost_usd_dollar=adjusted_cost,
+        cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=adjusted_cost,
         original_cost=reported_cost,
         discount_percent=discount_percent,
@@ -1887,7 +1928,10 @@ def completion_cost(
                         video_resolution=video_resolution,
                     )
                 elif call_type in _SPEECH_CALL_TYPES:
-                    prompt_characters = litellm.utils.count_characters(text=prompt)
+                    prompt_characters = _speech_billed_characters(
+                        completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
+                        prompt,
+                    )
                 elif call_type in _TRANSCRIPTION_CALL_TYPES:
                     # Check _hidden_params first (duration stored there to
                     # avoid polluting the response body), then fall back to
@@ -2338,6 +2382,15 @@ def response_cost_calculator(
                     )
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None and not custom_pricing:
+                        if _breakdown_total(litellm_logging_obj) != provider_response_cost:
+                            _store_cost_breakdown_in_logging_obj(
+                                litellm_logging_obj=litellm_logging_obj,
+                                prompt_tokens_cost_usd_dollar=0.0,
+                                completion_tokens_cost_usd_dollar=provider_response_cost,
+                                cost_for_built_in_tools_cost_usd_dollar=0.0,
+                                total_cost_usd_dollar=provider_response_cost,
+                                original_cost=provider_response_cost,
+                            )
                         return provider_response_cost
 
             response_cost = completion_cost(
@@ -2366,6 +2419,11 @@ def response_cost_calculator(
         raise e
 
 
+def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
+    reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
+    return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
+
+
 def _video_input_image_cost(
     model: str,
     custom_llm_provider: str | None,
@@ -3508,7 +3566,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
     lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
     session that should bill nothing fell through to the public rates instead.
     """
-    resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
+    resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
     entries: Final = (
         litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
         litellm.model_cost.get(model_name),
@@ -3704,9 +3762,9 @@ def handle_realtime_transcription_cost_calculation(
         return 0.0
 
     model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
+    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
     override_info: Final = (
-        _get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
+        _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
 
     total_cost = 0.0
@@ -3737,13 +3795,6 @@ def _get_transcription_model_name_from_results(
     return None
 
 
-def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
-    try:
-        return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
-    except Exception:
-        return None
-
-
 _TranscriptionRateKey: TypeAlias = Literal[
     "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
 ]

litellm/exceptions.py

@@ -9,6 +9,7 @@
 
 ## LiteLLM versions of the OpenAI Exception Types
 
+import datetime
 import enum
 from collections.abc import Mapping, Sequence
 from typing import Any, Final
@@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
     return None
 
 
+def _upstream_body(response: object) -> bytes:
+    if not isinstance(response, httpx.Response):
+        return b""
+    try:
+        return response.content
+    except httpx.ResponseNotRead:
+        return b""
+
+
+def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
+    if not isinstance(upstream, httpx.Response):
+        return httpx.Request(method="POST", url=fallback_url)
+    try:
+        upstream_request: Final = upstream.request
+    except RuntimeError:
+        return httpx.Request(method="POST", url=fallback_url)
+    return httpx.Request(
+        method=upstream_request.method,
+        url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
+    )
+
+
+def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
+    if not isinstance(upstream, httpx.Response):
+        return None
+    try:
+        return upstream.elapsed
+    except RuntimeError:
+        return None
+
+
+def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
+    response: Final = httpx.Response(
+        status_code=status_code,
+        stream=httpx.ByteStream(_upstream_body(upstream)),
+        request=_request_without_credentials(upstream, fallback_url),
+    )
+    _ = response.read()
+    response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
+    elapsed: Final = _upstream_elapsed(upstream)
+    if elapsed is not None:
+        response.elapsed = elapsed
+    return response
+
+
+_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
+
+
 _MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
 
 
@@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
         # headers stay reachable on `e.response.headers` for callers that
         # explicitly want them; only the proxy-supplied `headers=` kwarg
         # makes it onto `self.headers`.
-        _response_headers: Final = getattr(response, "headers", None) if response is not None else None
         self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
         # Mirrors FastAPI HTTPException.detail so the same instance can be
         # serialized through both the ProxyException and HTTPException paths.
         self.detail = detail if detail is not None else self.message
-        self.response = httpx.Response(
-            status_code=429,
-            headers=_response_headers,
-            content=response.content if response is not None else None,
-            request=httpx.Request(
-                method="POST",
-                url=" https://cloud.google.com/vertex-ai/",
-            ),
-        )
+        self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
         super().__init__(
             self.message,
             response=self.response,  # pyright: ignore[reportArgumentType]  # SDK accepts HTTPX at runtime
@@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
-        _response_headers: Final = getattr(response, "headers", None) if response is not None else None
-        self.response = httpx.Response(
-            status_code=self.status_code,
-            headers=_response_headers,
-            request=httpx.Request(
-                method="POST",
-                url=" https://cloud.google.com/vertex-ai/",
-            ),
-        )
+        self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
         super().__init__(
             self.message, response=self.response, body=None
         )  # Call the base class constructor with the parameters it needs
@@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
-        _response_headers: Final = getattr(response, "headers", None) if response is not None else None
-        self.response = httpx.Response(
-            status_code=self.status_code,
-            headers=_response_headers,
-            request=httpx.Request(
-                method="POST",
-                url=" https://cloud.google.com/vertex-ai/",
-            ),
-        )
+        self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
         super().__init__(
             self.message,
             response=self.response,  # pyright: ignore[reportArgumentType]  # SDK accepts HTTPX at runtime
@@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
-        _response_headers: Final = getattr(response, "headers", None) if response is not None else None
-        self.response = httpx.Response(
-            status_code=self.status_code,
-            headers=_response_headers,
-            request=httpx.Request(
-                method="POST",
-                url=" https://cloud.google.com/vertex-ai/",
-            ),
-        )
+        self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
         super().__init__(
             self.message,
             response=self.response,  # pyright: ignore[reportArgumentType]  # SDK accepts HTTPX at runtime
@@ -908,6 +924,7 @@ class APIError(openai.APIError):
         max_retries: int | None = None,
         num_retries: int | None = None,
         body: object | None = None,
+        response: httpx.Response | None = None,
     ):
         self.status_code = status_code
         self.message = f"litellm.APIError: {message}"
@@ -916,6 +933,8 @@ class APIError(openai.APIError):
         self.litellm_debug_info = litellm_debug_info
         self.max_retries = max_retries
         self.num_retries = num_retries
+        if response is not None:
+            self.response = response
         if request is None:
             request = httpx.Request(method="POST", url="https://api.openai.com/v1")
         super().__init__(self.message, request=request, body=body)

litellm/images/main.py

@@ -53,7 +53,7 @@ from litellm.main import (
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.images.main import ImageEditOptionalRequestParams
 from litellm.types.llms.openai import ImageGenerationRequestQuality
-from litellm.types.router import GenericLiteLLMParams
+from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
 from litellm.types.utils import (
     LITELLM_IMAGE_VARIATION_PROVIDERS,
     LlmProviders,
@@ -1010,6 +1010,7 @@ def image_edit(
             optional_params=dict(image_edit_request_params),
             litellm_params={
                 **image_edit_request_params,
+                **litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
                 "litellm_call_id": litellm_call_id,
                 "model_info": model_info,
                 "vertex_location": litellm_params.vertex_location,

litellm/images/utils.py

@@ -20,7 +20,7 @@ class ImageEditRequestUtils:
         image_edit_optional_params: ImageEditOptionalRequestParams,
         drop_params: bool | None = None,
         additional_drop_params: Collection[str] | None = None,
-    ) -> dict:
+    ) -> dict[str, object]:
         """
         Get optional parameters for the image edit API.
 

litellm/integrations/agentops/agentops.py

@@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
             "Connection": "keep-alive",
         }
 
-        client: Final = get_httpx_client()
-        try:
-            response: Final = client.post(
-                url=auth_endpoint,
-                headers=headers,
-                json={"api_key": api_key},
-                timeout=10,
-            )
+        response: Final = get_httpx_client().post(
+            url=auth_endpoint,
+            headers=headers,
+            json={"api_key": api_key},
+            timeout=10,
+        )
 
-            if response.status_code != 200:
-                raise Exception(f"Failed to fetch auth token: {response.text}")
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch auth token: {response.text}")
 
-            return _DECODED_JSON.validate_python(response.json())
-        finally:
-            client.close()
+        return _DECODED_JSON.validate_python(response.json())

litellm/integrations/anthropic_cache_control_hook.py

@@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import PromptManagementClient
+from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     cache_control_as_prompt_cache_breakpoint,
     with_prompt_cache_breakpoint,
@@ -115,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
     return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
 
 
-def _served_provider_for_model(model: str) -> str | None:
+def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
     from litellm.exceptions import BadRequestError
     from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
     try:
-        _, provider, _, _ = get_llm_provider(model=model)
+        _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
     except BadRequestError:
         return None
     return provider
@@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
 
 
 def _hosted_openai_dialect_flag(
-    model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
+    model: str,
+    custom_llm_provider: str | None,
+    resolve_provider: Callable[[str], str | None],
+    model_cost: Mapping[str, Mapping[str, object]] | None = None,
 ) -> bool | None:
     """
     Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
     """
     import litellm
 
-    exact_entry: Final = litellm.model_cost.get(model)
-    exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
+    cost_map: Final = (
+        cast(Mapping[str, Mapping[str, object]], litellm.model_cost)  # cast-ok: the live map holds catalog entries
+        if model_cost is None
+        else model_cost
+    )
+    exact_entry: Final = cost_map.get(model)
+    exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
     if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
         return None
     provider: Final = custom_llm_provider or resolve_provider(model)
     if provider is None or provider == "openai":
         return None
-    if isinstance(exact_entry, dict) and exact_provider == provider:
+    if isinstance(exact_entry, Mapping) and exact_provider == provider:
         return _hosted_entry_flag(exact_entry, lambda: provider)
     from litellm.utils import get_potential_model_names
 
@@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
         names["stripped_model_name"],
         names["provider_prefixed_model_name"],
     )
-    entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
-    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
+    entries: Final = (cost_map.get(candidate) for candidate in candidates)
+    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
     return next((flag for flag in flags if flag is not None), None)
 
 
@@ -527,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if model is None:
             return False
         hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
+            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
         )
         if hosted_flag is not None:
             return hosted_flag
@@ -538,8 +547,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return prompt_cache_options is not None or targets_openai_api(api_base)
 
     @staticmethod
-    def _resolve_provider(model: str) -> str | None:
-        return _served_provider_for_model(model)
+    def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
+        return (
+            _hosted_openai_dialect_flag(
+                model,
+                custom_llm_provider,
+                AnthropicCacheControlHook._resolve_provider,
+                GetModelCostMap.loaded_model_cost_map(),
+            )
+            is True
+        )
+
+    @staticmethod
+    def _resolve_provider(model: str, api_base: object = None) -> str | None:
+        return _served_provider_for_model(model, api_base)
 
     @staticmethod
     def count_request_cache_breakpoints(

litellm/integrations/arize/arize_phoenix_client.py

@@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 import urllib.parse
 from typing import Final
 
+from pydantic import BaseModel, ConfigDict
+
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+class _PromptVersionResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
+
+    data: dict[str, object] | None = None
+
+
 def _sanitize_id(identifier: str) -> str:
     """Reject path traversal characters and URL-encode the identifier."""
     if any(c in identifier for c in ("/", "\\", "#", "?")):
@@ -71,8 +79,7 @@ class ArizePhoenixClient:
             response: Final = self.http_handler.get(url, headers=self.headers)
             response.raise_for_status()
 
-            data: Final = response.json()
-            return data.get("data")
+            return _PromptVersionResponse.model_validate(response.json()).data
 
         except Exception as e:
             # Check if it's an HTTP error

litellm/integrations/custom_guardrail.py

@@ -9,6 +9,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
 
 import httpx
+from pydantic import TypeAdapter
 
 from litellm._internal_context import with_service_target
 from litellm._logging import verbose_logger
@@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
     )
 
 
+_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
+
+
+def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
+    if not isinstance(metadata, dict):
+        return []
+    return metadata.get("guardrails") or []
+
+
 class CustomGuardrail(CustomLogger):
     # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
     use_native_during_call_hook: ClassVar[bool] = False
@@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
                 return True
             raise
 
-    def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
+    def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
         """
-        Returns the guardrail(s) to be run from the metadata or root
+        Returns the guardrail(s) to be run from the root, else from both metadata fields together
         """
 
         if "guardrails" in data:
             return data["guardrails"]
-        # Check both metadata locations. For regular endpoints move_guardrails_to_metadata
-        # writes to "metadata"; for thread/assistant endpoints it writes to
-        # "litellm_metadata". We check the one that actually contains the "guardrails"
-        # key so that a non-empty litellm_metadata without guardrails does not shadow
-        # the merged list stored in metadata (which would cause team guardrails to be
-        # silently skipped while default_on=True policy guardrails still fire).
-        for meta_key in ("metadata", "litellm_metadata"):
-            meta = data.get(meta_key) or {}
-            if isinstance(meta, dict) and "guardrails" in meta:
-                return meta.get("guardrails") or []
-        return []
+        request_fields: Final = _REQUEST_FIELDS.validate_python(data)
+        return [
+            *_guardrails_listed_in(request_fields.get("metadata")),
+            *_guardrails_listed_in(request_fields.get("litellm_metadata")),
+        ]
 
     def _guardrail_is_in_requested_guardrails(
         self,
-        requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
+        requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
     ) -> bool:
         for _guardrail in requested_guardrails:
             if isinstance(_guardrail, dict):

litellm/integrations/langfuse/langfuse_otel.py

@@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
+    from litellm.proxy._types import UserAPIKeyAuth
+
     Span = _Span
 else:
     Span = Any
@@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
     )
 
 
+def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
+    """Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
+    every call but one whose logging payload shows it never reached a provider."""
+    from litellm.integrations.otel.model.metadata import LLMCallEvent
+
+    if not _a_destination_carries_the_trace():
+        return False
+    call: Final = LLMCallEvent.from_dict(kwargs)
+    return call.payload is None or call.reached_a_provider
+
+
+def _a_destination_carries_the_trace() -> bool:
+    """Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
+    trace."""
+    from litellm.integrations.otel.model.config import ExporterOwner
+    from litellm.integrations.otel.plumbing.context import destination_backends
+
+    return ExporterOwner.LANGFUSE_OTEL in destination_backends()
+
+
 class LangfuseOtelLogger(OpenTelemetry):
     def __init__(self, config=None, *args, **kwargs):
         # Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
@@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
         """
         return None
 
+    def _emit_once(
+        self,
+        kwargs: dict[str, object],  # mutable-ok: the v1 dedupe marker is written into the request's kwargs
+        *scope: object,
+    ) -> bool:
+        """Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
+        return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope)  # pyright: ignore[reportUnknownMemberType]  # the v1 hook takes a bare dict
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],  # mutable-ok: the v1 hook contract
+        original_exception: Exception,
+        user_api_key_dict: "UserAPIKeyAuth",
+        traceback_str: str | None = None,
+    ) -> None:
+        """Record the proxy failure of a request no Langfuse destination carries."""
+        if not _a_destination_carries_the_trace():
+            await super().async_post_call_failure_hook(  # pyright: ignore[reportUnknownMemberType]  # the v1 hook takes a bare dict
+                request_data, original_exception, user_api_key_dict, traceback_str
+            )
+
     async def async_service_success_hook(self, *args, **kwargs):
         """
         Langfuse should not receive service success logs.

litellm/integrations/opentelemetry.py

@@ -72,7 +72,7 @@ if TYPE_CHECKING:
     )
     from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
-    Span = _Span | Any
+    Span = _Span
     Tracer = _Tracer
     Context = _Context
     SpanExporter = _SpanExporter
@@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             span = None
             # Only set attributes if the span is still recording (not closed)
             # Note: parent_span is guaranteed to be not None here
-            if hasattr(parent_span, "set_status"):
+            if parent_span is not None and hasattr(parent_span, "set_status"):
                 parent_span.set_status(Status(StatusCode.OK))
                 self.set_attributes(parent_span, kwargs, response_obj)
             # Raw-request as direct child of parent_span
@@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             # record error on parent span (keeps hierarchy shallow)
             # Only set attributes if the span is still recording (not closed)
             # Note: parent_otel_span is guaranteed to be not None here
-            if parent_otel_span.is_recording():
+            if parent_otel_span is not None and parent_otel_span.is_recording():
                 self.set_attributes(parent_otel_span, kwargs, response_obj)
                 parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
                 parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))

litellm/integrations/otel/emitter.py

@@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
 
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
-from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
+from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
 from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
-from litellm.integrations.otel.model.config import OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
@@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
     mcp_tool_call_span_name,
     service_span_name,
 )
+from litellm.integrations.otel.plumbing.context import destination_backends
 from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
 from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
 
@@ -168,6 +169,9 @@ class SpanEmitter:
         self._mappers: list[AttributeMapper] = (
             list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
         )
+        self._langfuse: Final = (
+            None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
+        )
         # Bounded LRU (ordered by insertion / most-recent touch). Storing keys
         # only — the value is unused — so it behaves like a capped set.
         self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
@@ -295,17 +299,27 @@ class SpanEmitter:
             )
             else None
         )
-        mapped: Final = MappingProxyType(
-            {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
+        configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
+        for_langfuse: Final = (
+            dict(
+                sorted(
+                    self._langfuse.map(data).items(),
+                    key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
+                )
+            )
+            if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
+            else _NO_ATTRIBUTES
         )
+        mapped: Final = MappingProxyType({**configured, **for_langfuse})
         stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
         reserved: Final = len(stamped_later.keys() - mapped.keys())
         overwrites: Final = carried_keys(span) & mapped.keys()
-        # The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
+        # The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
+        # so a span at its attribute limit loses nothing else to them.
         fitted: Final = fit_indexed_messages(
             MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
             attribute_budget(span, reserved),
-            LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
+            LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
         )
         for key, value in mapped.items():
             if key in overwrites or key in fitted:

litellm/integrations/otel/logger.py

@@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
         tracer_provider: TracerProvider | None = None,
         logger_provider: LoggerProvider | None = None,
         meter_provider: "MeterProvider | None" = None,
+        console_default: bool = False,
         **kwargs: Unpack[_CustomLoggerOptions],
     ) -> None:
         super().__init__(**kwargs)
@@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
         self._tracer_provider: TracerProvider = (
             tracer_provider
             if tracer_provider is not None
-            else build_tracer_provider(self.config, tenant_overrides=True)
+            else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
         )
         self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
         self._metrics_recorder = self._init_metrics(meter_provider)
@@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
             mappers=resolve_mappers(self.config.mapper_names),
             event_recorder=self._init_events(logger_provider),
         )
-        self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
+        self._tenant_tracers = TenantTracerCache(
+            self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
+        )
         self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
         self._init_otel_logger_on_litellm_proxy()
 
@@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
         # gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
         # hit never hands off to a provider, so ``cache_hit`` substitutes for the
         # upstream stamp, since the payload proves the call completed.
-        if carrier is None and (
-            call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
-        ):
+        if carrier is None and not call.reached_a_provider:
             return None
         try:
             return self._finish_carrier(carrier, call, start_time, end_time)
@@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
     logger_provider: LoggerProvider | None = None,
     meter_provider: "MeterProvider | None" = None,
     settings: Mapping[str, object] = MappingProxyType({}),
+    console_default: bool = False,
 ) -> OpenTelemetryV2:
     return _logger_class(config)(
         config=config,
@@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
         tracer_provider=tracer_provider,
         logger_provider=logger_provider,
         meter_provider=meter_provider,
+        console_default=console_default,
         **settings,
     )
 

litellm/integrations/otel/mappers/langfuse.py

@@ -103,6 +103,18 @@ class LangfuseMapper:
         ),
     }
 
+    #: Every attribute this mapper can write on a model call.
+    LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
+        {
+            *_LLM_CALL_ATTRS,
+            *_BLOB_ATTRS,
+            LANGFUSE_TRACE_NAME,
+            LANGFUSE_TRACE_USER_ID,
+            LANGFUSE_TRACE_SESSION_ID,
+            LANGFUSE_TRACE_TAGS,
+        }
+    )
+
     def map(self, data: SpanData) -> AttributeMap:
         match data:
             case LLMCallSpanData():

litellm/integrations/otel/model/config.py

@@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
     )
 
 
+def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
+    """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
+
+    No field set is what says the operator asked for nothing: an exporter they did
+    configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
+    and so does the gated spec ``credential_gated_exporters`` appends, which would
+    otherwise eat itself when one preset layers onto another.
+    """
+    return not spec.model_fields_set
+
+
 class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
     def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
         super().__init__(settings_cls)

litellm/integrations/otel/model/metadata.py

@@ -272,6 +272,11 @@ class LLMCallEvent:
     trace: TraceControls
     session_id: str | None
 
+    @property
+    def reached_a_provider(self) -> bool:
+        """Whether the call was handed to a provider or answered from the response cache."""
+        return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
+
     @classmethod
     def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
         raw_payload: Final = kwargs.get("standard_logging_object")

litellm/integrations/otel/model/spans.py

@@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_LensReview",
         "LiteLLM_LensIngestionKey",
         "LiteLLM_LensWorker",
+        "LiteLLM_SubscriptionAccountTable",
         "LiteLLM_LensSignalConfig",
         "LiteLLM_LensTraceSignal",
     )

litellm/integrations/otel/plumbing/providers.py

@@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
     LANGFUSE_OBSERVATION_OUTPUT,
     LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
     LANGFUSE_TRACE_NAME,
+    LangfuseMapper,
 )
 from litellm.integrations.otel.model.config import (
     CaptureMessageContent,
@@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
     OpenTelemetryV2Config,
     capture_message_content_from_env,
     captures_span_content,
+    is_unconfigured_placeholder,
 )
 from litellm.integrations.otel.model.semconv import (
     DB,
@@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
     return scope is not None and scope.name == LITELLM_TRACER_NAME
 
 
-def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
-    """``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
+def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
+    """``span`` without the Langfuse attributes only a Langfuse destination asked for.
 
-    Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
+    That is the request metadata the Langfuse mapper writes and, on a model call written by
+    a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
+    every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
+    destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
+    its own code set.
     """
     attributes: Final = span.attributes or _NO_ATTRIBUTES
-    if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
+    dropped: Final = (
+        LANGFUSE_REQUEST_METADATA_ATTRIBUTES
+        if langfuse_mapped or not is_llm_call_span(span)
+        else LangfuseMapper.LLM_CALL_ATTRIBUTES
+    )
+    if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
         return span
     return _CountedSpanView(
-        span,
-        MappingProxyType(
-            {key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
-        ),
+        span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
     )
 
 
@@ -658,6 +666,7 @@ def _for_destination(
     span: ReadableSpan,
     destination: "OtelDestination",
     default_capture: str,
+    langfuse_mapped: bool,
 ) -> ReadableSpan:
     """The view of ``span`` a tenant destination receives.
 
@@ -672,13 +681,16 @@ def _for_destination(
     and the query string of the request URL, which can hold the same key. A
     destination whose effective setting disables span content loses the prompt and
     response content too, and a destination for any backend but Langfuse loses the
-    request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
-    stays, so the tenant still gets the whole trace tree.
+    Langfuse attributes only a Langfuse destination asked for (see
+    :func:`without_destination_langfuse_attributes`). The span itself stays, so the
+    tenant still gets the whole trace tree.
     """
     extra: Final = destination.resource_attributes
     defaults: Final = destination.resource_defaults
     source: Final = (
-        span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
+        span
+        if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
+        else without_destination_langfuse_attributes(span, langfuse_mapped)
     )
     attributes: Final = source.attributes or _NO_ATTRIBUTES
     database: Final = _is_database_span(attributes)
@@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
         default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
         pending_drains: int = _MAX_PENDING_DRAINS,
         drain_pool: _DrainPool | None = None,
+        langfuse_mapped: bool = True,
     ) -> None:
         self._operator_sinks: Final = operator_sinks
+        self._langfuse_mapped: Final = langfuse_mapped
         self._excluded_db_systems: Final = excluded_db_systems
         self._default_capture: Final = default_capture
         self._default_capture_by_backend: Final = default_capture_by_backend
@@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
         try:
             processor.on_end(
                 _scoped(
-                    _for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
+                    _for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
+                    destination.span_scope,
                 )
             )
         except Exception as exc:  # noqa: BLE001  # one destination's failure must not cost the others their span
@@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
         return self._inner.force_flush(timeout_millis)
 
 
-class _LangfuseRequestMetadataFilter(SpanProcessor):
-    """Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
+class _DestinationLangfuseAttributeFilter(SpanProcessor):
+    """Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
+    exports them."""
 
-    def __init__(self, inner: SpanProcessor) -> None:
+    def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
         self._inner: Final = inner
+        self._langfuse_mapped: Final = langfuse_mapped
 
     def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
         self._inner.on_start(span, parent_context)
 
     def on_end(self, span: ReadableSpan) -> None:
-        self._inner.on_end(without_langfuse_request_metadata(span))
+        self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
 
     def shutdown(self) -> None:
         self._inner.shutdown()
@@ -1491,13 +1508,18 @@ def _spec_processor(
     spec: ExporterSpec,
     use_simple_processor: bool | None,
     strip_message_content: bool,
-    strip_langfuse_request_metadata: bool,
+    strip_destination_langfuse_attributes: bool,
+    langfuse_mapped: bool,
 ) -> SpanProcessor:
     exporting: Final = _processor_for(
         _exporter_from_spec(spec),
         (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
     )
-    scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
+    scoped: Final = (
+        _DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
+        if strip_destination_langfuse_attributes
+        else exporting
+    )
     return _MessageContentFilter(scoped) if strip_message_content else scoped
 
 
@@ -1530,6 +1552,7 @@ def build_tracer_provider(
     use_simple_processor: bool | None = None,
     tenant_overrides: bool = False,
     strip_message_content: bool = False,
+    console_default: bool = False,
 ) -> TracerProvider:
     """Build the shared :class:`TracerProvider`.
 
@@ -1548,11 +1571,16 @@ def build_tracer_provider(
 
     ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
     alone; a collector or any other backend in the same config keeps the full tree.
-    In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
-    goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
+    Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
+    Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
+    which any logger writes for a request with a Langfuse destination.
     ``strip_message_content`` removes prompt and response content before every
     exporter, for a routed provider whose spans can carry content another
     destination asked for.
+
+    The console spec ``config._normalize`` folds in when nothing was configured
+    prints only under ``console_default``, which the ``otel`` callback passes: a
+    vendor preset or the proxy's background global logger was never asked to print.
     """
     provider: Final = TracerProvider(resource=build_resource(config))
     provider.sampler = _RequestExclusionSampler(provider.sampler)
@@ -1564,19 +1592,20 @@ def build_tracer_provider(
         provider.add_span_processor(_processor_for(exporter, use_simple_processor))
         return provider
 
+    langfuse_mapped: Final = _maps_langfuse(config)
     # ``config._normalize`` guarantees at least one spec (it folds the top-level
     # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
-    writes_langfuse: Final = "langfuse" in config.mapper_names or any(
-        spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
-    )
     for spec in config.exporters:
         if spec.requires_headers and not spec.headers:
             continue
+        if is_unconfigured_placeholder(spec) and not console_default:
+            continue
         processor = _spec_processor(
             spec,
             use_simple_processor,
             strip_message_content,
-            strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
+            strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
+            langfuse_mapped=langfuse_mapped,
         )
         owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
         scope = _operator_scope(config, spec)
@@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
     so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
     config per v2 logger since each keeps its own provider and still writes its
     account, so an additive destination pointing at any of them is delivered once
-    rather than twice. ``excluded_db_systems`` only filters what the fan-out
-    delivers, never the operator's own exporters.
+    rather than twice. The first is the config of the logger that owns ``provider``,
+    whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
+    the fan-out delivers, never the operator's own exporters.
     """
     with _FAN_OUT_ATTACH_LOCK:
         if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
@@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
                 excluded_db_systems=excluded_db_systems,
                 default_capture=_operator_capture_mode(configs),
                 default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
+                langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
             )
         )
 
 
+def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
+    """Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
+    return "langfuse" in config.mapper_names
+
+
 def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
     """The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
     return next(

litellm/integrations/otel/plumbing/routing.py

@@ -163,10 +163,12 @@ class TenantTracerCache:
         config: OpenTelemetryV2Config,
         callback_name: str | None,
         tracer_name: str,
+        console_default: bool = False,
     ) -> None:
         self._config = config
         self._callback_name = callback_name
         self._tracer_name = tracer_name
+        self._console_default = console_default
         # Guards the three mutable structures below: ``pre_call`` can run on
         # thread-pool workers concurrently with the event loop, so cache
         # updates, span counts, and retirement must be atomic.
@@ -284,6 +286,7 @@ class TenantTracerCache:
         built: Final = build_tracer_provider(
             self._routed_config(credential_headers, project_headers, endpoint, service_name),
             strip_message_content=not self._config.capture_span_content,
+            console_default=self._console_default,
         )
         self._providers[cache_key] = built
         return built

litellm/integrations/otel/presets/destinations.py

@@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
 _NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
 
 
-def _arize_project() -> Mapping[str, str]:
+def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
     from litellm.integrations.arize.arize import ArizeLogger
 
     project: Final = ArizeLogger.get_arize_config().project_name
@@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
     return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
 
 
-#: Resource attributes the operator chose for a backend, applied to every export the
-#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
-_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
-    {"arize": _arize_project}
+def _stands_in_for_the_legacy_langfuse_logger() -> bool:
+    """Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
+    of its own sends for a key's or team's ``langfuse_otel``."""
+    return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
+
+
+def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
+    """The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
+    logger wrote it."""
+    from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
+
+    if not _stands_in_for_the_legacy_langfuse_logger():
+        return _NO_ATTRS
+    environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
+    return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
+
+
+#: Resource attributes a backend gets on every export the team receives, over the
+#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
+_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
+    MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
 )
 
 #: Resource attributes a backend needs on every export to accept it, filled only where
@@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
 )
 
 
-def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
+def _resource_attributes(
+    callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
+) -> Mapping[str, str]:
     backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
     return MappingProxyType(
         {
             **({"service.name": service_name} if service_name else {}),
-            **(backend() if backend is not None else {}),
+            **(backend(params) if backend is not None else {}),
         }
     )
 
@@ -214,7 +233,7 @@ def destination_for(
     return OtelDestination(
         endpoint=endpoint,
         headers=MappingProxyType(dict(headers)),
-        resource_attributes=_resource_attributes(callback_name, service_name),
+        resource_attributes=_resource_attributes(callback_name, service_name, params),
         resource_defaults=_resource_defaults(callback_name),
         callback_name=callback_name,
         protocol=protocol,

litellm/integrations/otel/presets/utils.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 from typing import Final
 
-from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
+from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
 
 
 def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
@@ -35,14 +35,3 @@ def credential_gated_exporters(
         *(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
         ExporterSpec(owner=owner, requires_headers=True),
     )
-
-
-def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
-    """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
-
-    No field set is what says the operator asked for nothing: an exporter they did
-    configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
-    and so does the gated spec this module appends, which would otherwise eat itself
-    when one preset layers onto another.
-    """
-    return not spec.model_fields_set

litellm/integrations/websearch_interception/handler.py

@@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
             (t for t in tools if is_anthropic_native_web_search_tool(t)),
             None,
         )
+        emit_native_blocks: Final = native_tool is not None or (
+            kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
+        )
 
         outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
         search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
 
         content: Final[list[dict[str, object]]] = []
-        if native_tool is not None:
+        if emit_native_blocks:
             tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
-            tool_name: Final = native_tool.get("name") or "web_search"
+            tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
             content.append(
                 {
                     "type": "server_tool_use",
@@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
                 }
             )
             content.append(
-                WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
+                WebSearchTransformation.build_web_search_outcome_block(
+                    tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
+                )
             )
-        # Keep the text block so non-native short-circuit callers (Claude Code,
-        # github_copilot, etc.) see the same payload they always have.
         content.append({"type": "text", "text": search_result_text})
 
         response: Final[dict[str, object]] = {
@@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         verbose_logger.debug(
             "WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
             len(search_result_text),
-            native_tool is not None,
+            emit_native_blocks,
         )
         return response
 

litellm/integrations/websearch_interception/transformation.py

@@ -412,6 +412,7 @@ class WebSearchTransformation:
     def build_web_search_tool_result_block(
         tool_use_id: str,
         search_response: SearchResponse | None,
+        with_snippets: bool = True,
     ) -> dict[str, object]:
         """
         Build an Anthropic-native ``web_search_tool_result`` content block.
@@ -445,6 +446,8 @@ class WebSearchTransformation:
                 ``litellm.asearch()``. If None or empty, the block is still
                 emitted with an empty result list (signals "search ran, no
                 results" rather than "search did not run").
+            with_snippets: False when a text block in the same reply already
+                carries the snippets.
         """
         items: Final[list[dict[str, object]]] = []
         if search_response is not None:
@@ -460,7 +463,7 @@ class WebSearchTransformation:
                         "title": title,
                         "page_age": page_age,
                         "encrypted_content": "",
-                        "snippet": getattr(r, "snippet", "") or "",
+                        **({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
                     }
                 )
         return {
@@ -481,12 +484,15 @@ class WebSearchTransformation:
         }
 
     @staticmethod
-    def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
+    def build_web_search_outcome_block(
+        tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
+    ) -> dict[str, object]:
         match outcome:
             case SearchSucceeded(response=response):
                 return WebSearchTransformation.build_web_search_tool_result_block(
                     tool_use_id=tool_use_id,
                     search_response=response,
+                    with_snippets=with_snippets,
                 )
             case SearchFailed(error_code=error_code):
                 return WebSearchTransformation.build_web_search_tool_result_error_block(

litellm/litellm_core_utils/core_helpers.py

@@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
     return getattr(chunk, "usage", None)
 
 
-USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
+PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
+
+
+UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
+
+
+def _numeric(value: object) -> float | None:
+    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
+
+
+def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
+    """Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
+    ``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
+    reading litellm's bill back as if the upstream had reported it."""
+    hidden: Final[object] = getattr(response, "_hidden_params", None)
+    if isinstance(hidden, dict):
+        hidden_params: Final = cast(dict[str, object], hidden)  # cast-ok: hidden params ride on the response as a dict
+        upstream: Final = (
+            hidden_params[UPSTREAM_USAGE_COST]
+            if UPSTREAM_USAGE_COST in hidden_params
+            else _numeric(getattr(usage, "cost", None))
+        )
+        hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
+    setattr(usage, "cost", bill)
 
 
 def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
-    if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
+    """The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
+    usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
+    event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
+    if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
         return None
-    cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
+    cost: Final = (
+        _numeric(hidden[UPSTREAM_USAGE_COST])
+        if hidden is not None and UPSTREAM_USAGE_COST in hidden
+        else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    )
     return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
 
 

litellm/litellm_core_utils/env_utils.py

@@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
     Values outside the range fall back to the default and warn, so a misconfigured knob can
     neither crash the caller nor silently change the meaning of what it computes.
     """
-    value: Final = get_env_int(env_var, default)
+    return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
+
+
+def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
+    """Float counterpart of `get_env_int_in_range`."""
+    return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
+
+
+def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
     if minimum <= value <= maximum:
         return value
     logging.getLogger("LiteLLM").warning(

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -37,6 +37,7 @@ from ..exceptions import (
     ServiceUnavailableError,
     Timeout,
     UnprocessableEntityError,
+    error_response_without_credentials,
 )
 
 
@@ -305,6 +306,16 @@ def _litellm_proxy_response(
     )
 
 
+def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
+    if not isinstance(response, httpx.Response):
+        return response
+    return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
+
+
+def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
+    return response if response is not None and (response.headers or response.content) else None
+
+
 def _map_openai_exception(
     *,
     model: str,
@@ -315,7 +326,7 @@ def _map_openai_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
-    response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
+    response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
     if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
         raise ContentPolicyViolationError(
             message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
@@ -378,6 +389,7 @@ def _map_openai_exception(
             model=model,
             llm_provider=custom_llm_provider,
             litellm_debug_info=extra_information,
+            response=_upstream_reply(response),
         )
     elif (
         ("invalid_request_error" in error_str and "content_policy_violation" in error_str)
@@ -449,6 +461,7 @@ def _map_openai_exception(
             model=model,
             llm_provider=custom_llm_provider,
             body=getattr(original_exception, "body", None),
+            response=_upstream_reply(response),
         )
     elif "Request too large" in error_str:
         raise RateLimitError(
@@ -479,6 +492,7 @@ def _map_openai_exception(
             model=model,
             request=_request,
             litellm_debug_info=extra_information,
+            response=_upstream_reply(response),
         )
     elif hasattr(original_exception, "status_code"):
         if original_exception.status_code == 400:
@@ -512,6 +526,7 @@ def _map_openai_exception(
                 model=model,
                 llm_provider=custom_llm_provider,
                 litellm_debug_info=extra_information,
+                response=_upstream_reply(response),
             )
         elif original_exception.status_code == 422:
             raise BadRequestError(
@@ -563,6 +578,7 @@ def _map_openai_exception(
                 llm_provider=custom_llm_provider,
                 litellm_debug_info=extra_information,
                 exception_status_code=original_exception.status_code,
+                response=_upstream_reply(response),
             )
         else:
             raise APIError(
@@ -572,6 +588,7 @@ def _map_openai_exception(
                 model=model,
                 request=getattr(original_exception, "request", None),
                 litellm_debug_info=extra_information,
+                response=_upstream_reply(response),
             )
     else:
         # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@@ -2019,6 +2036,7 @@ def _map_azure_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    response: Final = _without_credentials(getattr(original_exception, "response", None))
     message = get_error_message(error_obj=original_exception)
     if message is None:
         if hasattr(original_exception, "message"):
@@ -2054,7 +2072,7 @@ def _map_azure_exception(
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
         )
     elif "DeploymentNotFound" in error_str:
         raise NotFoundError(
@@ -2062,7 +2080,7 @@ def _map_azure_exception(
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
         )
     elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
         error_str
@@ -2076,6 +2094,7 @@ def _map_azure_exception(
             model=model,
             extra_information=extra_information,
             original_exception=original_exception,
+            response=response,
         )
     elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
         helpful_message: Final = (
@@ -2094,7 +2113,7 @@ def _map_azure_exception(
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
             body=getattr(original_exception, "body", None),
         )
     elif getattr(original_exception, "status_code", None) in (None, 400) and (
@@ -2106,7 +2125,7 @@ def _map_azure_exception(
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
             body=getattr(original_exception, "body", None),
         )
     elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
@@ -2115,7 +2134,7 @@ def _map_azure_exception(
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
             body=getattr(original_exception, "body", None),
         )
     elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
@@ -2124,7 +2143,7 @@ def _map_azure_exception(
             llm_provider=custom_llm_provider,
             model=model,
             litellm_debug_info=extra_information,
-            response=getattr(original_exception, "response", None),
+            response=response,
         )
     elif "Connection error" in error_str:
         raise APIConnectionError(
@@ -2140,7 +2159,7 @@ def _map_azure_exception(
                 llm_provider="azure",
                 model=model,
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 body=getattr(original_exception, "body", None),
             )
         elif original_exception.status_code == 401:
@@ -2149,7 +2168,7 @@ def _map_azure_exception(
                 llm_provider="azure",
                 model=model,
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 404:
             raise NotFoundError(
@@ -2157,7 +2176,7 @@ def _map_azure_exception(
                 llm_provider="azure",
                 model=model,
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 408:
             raise Timeout(
@@ -2165,6 +2184,7 @@ def _map_azure_exception(
                 model=model,
                 litellm_debug_info=extra_information,
                 llm_provider="azure",
+                response=_upstream_reply(response),
             )
         elif original_exception.status_code == 422:
             raise BadRequestError(
@@ -2172,7 +2192,7 @@ def _map_azure_exception(
                 model=model,
                 llm_provider="azure",
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 429:
             raise RateLimitError(
@@ -2180,7 +2200,7 @@ def _map_azure_exception(
                 model=model,
                 llm_provider="azure",
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 502:
             raise BadGatewayError(
@@ -2188,7 +2208,7 @@ def _map_azure_exception(
                 model=model,
                 llm_provider="azure",
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 503:
             raise ServiceUnavailableError(
@@ -2196,7 +2216,7 @@ def _map_azure_exception(
                 model=model,
                 llm_provider="azure",
                 litellm_debug_info=extra_information,
-                response=getattr(original_exception, "response", None),
+                response=response,
             )
         elif original_exception.status_code == 504:  # gateway timeout error
             raise Timeout(
@@ -2205,6 +2225,7 @@ def _map_azure_exception(
                 litellm_debug_info=extra_information,
                 llm_provider="azure",
                 exception_status_code=original_exception.status_code,
+                response=_upstream_reply(response),
             )
         else:
             raise APIError(
@@ -2214,6 +2235,7 @@ def _map_azure_exception(
                 litellm_debug_info=extra_information,
                 model=model,
                 request=httpx.Request(method="POST", url="https://openai.com/"),
+                response=_upstream_reply(response),
             )
     else:
         # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@@ -2589,17 +2611,20 @@ def exception_type(
                     model=model,
                     llm_provider=custom_llm_provider,
                     litellm_debug_info=extra_information,
+                    response=_upstream_reply(
+                        _without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
+                    ),
                 )
 
             if (
                 custom_llm_provider == "litellm_proxy"
             ):  # handle special case where calling litellm proxy + exception str contains error message
                 extract_and_raise_litellm_exception(
-                    response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
+                    response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
                     error_str=error_str,
                     model=model,
                     custom_llm_provider=custom_llm_provider,
-                    body=getattr(original_exception, "body", None),
+                    body=getattr(mappable_exception, "body", None),
                 )
             if (
                 custom_llm_provider == "openai"

litellm/litellm_core_utils/get_litellm_params.py

@@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_user_session",
             "fireworks_forward_user_id",
             "responses_api_version",
+            "allowed_openai_params",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
         }
     )

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -292,6 +292,10 @@ def get_llm_provider(
                     elif endpoint == "api.deepinfra.com/v1/openai":
                         custom_llm_provider = "deepinfra"
                         dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
+                    elif endpoint == "api.clfaigateway.dev/v1":
+                        custom_llm_provider = "clf_ai_gateway"  # rebind-ok: endpoint inference, like every branch
+                        clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
+                        dynamic_api_key = api_key or clf_env_key
                     elif endpoint == "api.mistral.ai/v1":
                         custom_llm_provider = "mistral"
                         dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
@@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "clf_ai_gateway":
+        (
+            api_base,  # rebind-ok: resolved base/key are what this function returns, like every branch
+            dynamic_api_key,  # rebind-ok: see above
+        ) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "empower":
         api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
         dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")

litellm/litellm_core_utils/health_check_helpers.py

@@ -3,6 +3,8 @@ Helper functions for health check calls.
 """
 
 import base64
+import itertools
+import re
 from collections.abc import Awaitable, Callable, Mapping
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal
@@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
 )
 
 ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
+WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
+WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
+BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
 EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
     {"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
 )
@@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
     return native(model, custom_llm_provider)
 
 
+def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
+    from litellm.types.utils import LlmProviders
+    from litellm.utils import ProviderConfigManager
+
+    try:
+        model_info: Final = ProviderConfigManager.get_provider_model_info(
+            model=None, provider=LlmProviders(custom_llm_provider)
+        )
+        return None if model_info is None else model_info.get_base_model(model)
+    except (ValueError, NotImplementedError):
+        return None
+
+
+def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
+    if custom_llm_provider == "bedrock":
+        from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
+
+        return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
+    base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
+    return base_model is not None and not base_model.startswith(f"{segment}/")
+
+
+def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
+    from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
+    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+
+    provider_prefix: Final = f"{custom_llm_provider}/"
+    segments: Final = wildcard.removeprefix(provider_prefix).split("/")
+    route_segments: Final = tuple(
+        itertools.takewhile(
+            lambda segment_index: _is_route_segment(
+                custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
+            ),
+            range(len(segments) - 1),
+        )
+    )
+    route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
+    model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
+    ranked_model_ids: Final = tuple(
+        (ranked_model.model.removeprefix(provider_prefix), ranked_model)
+        for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
+    )
+    served_model_ids: Final = sorted(
+        (ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
+        key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
+    )
+    served_probes: Final = itertools.islice(
+        dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
+        WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
+    )
+    provider_probes: Final = (
+        f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
+    )
+    return tuple(dict.fromkeys((*served_probes, *provider_probes)))
+
+
 class HealthCheckHelpers:
     @staticmethod
     async def ahealth_check_wildcard_models(
@@ -99,23 +160,16 @@ class HealthCheckHelpers:
         litellm_logging_obj: "Logging",
     ) -> dict:
         from litellm import acompletion
-        from litellm.litellm_core_utils.llm_request_utils import (
-            pick_cheapest_chat_models_from_llm_provider,
-        )
 
-        # this is a wildcard model, we need to pick a random model from the provider
-        cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
-        if len(cheapest_models) == 0:
+        probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
+        if len(probe_models) == 0:
             raise Exception(
-                f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
+                f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
+                "no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
             )
-        if len(cheapest_models) > 1:
-            fallback_models = cheapest_models[1:]  # Pick the last 2 models from the shuffled list
-        else:
-            fallback_models = None
-        model_params["model"] = cheapest_models[0]
+        model_params["model"] = probe_models[0]
         model_params["litellm_logging_obj"] = litellm_logging_obj
-        model_params["fallbacks"] = fallback_models
+        model_params["fallbacks"] = list(probe_models[1:]) or None
         model_params["max_tokens"] = model_params.get("max_tokens", 16)  # GPT-5 models require max_output_tokens >= 16
         await acompletion(**model_params)
         return {}

litellm/litellm_core_utils/litellm_logging.py

@@ -1,6 +1,7 @@
 # What is this?
 ## Common Utility file for Logging handler
 # Logging function -> log the exact model details + what's being sent | Non-Blocking
+import asyncio
 import copy
 import datetime
 import functools
@@ -96,6 +97,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
     MODEL_ACCESS_GROUP_METADATA_KEY,
     is_unbilled_non_inference_call,
 )
+from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost, PreparedCost
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     bedrock_guardrail_trace,
     cost_breakdown_with_guardrail,
@@ -200,6 +202,7 @@ from litellm.types.utils import (
     TextCompletionResponse,
     TranscriptionResponse,
     Usage,
+    batch_chat_usage,
 )
 from litellm.types.videos.main import VideoObject
 from litellm.utils import get_base_model_from_metadata, print_verbose
@@ -813,6 +816,11 @@ class Logging(LiteLLMLoggingBaseClass):
 
         # Initialize cost breakdown field
         self.cost_breakdown: CostBreakdown | None = None
+        self.finalized_cost: FinalizedCost | None = None
+        self.prepared_cost: PreparedCost | None = None
+        self.cost_allowance_discount: float | None = None
+        self.cost_finalization_lock = asyncio.Lock()
+        self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
         self.billed_token_rates: BilledTokenRates | None = None
         self._provider_reported_cost: float | None = None
 
@@ -874,6 +882,11 @@ class Logging(LiteLLMLoggingBaseClass):
             "current_attempt_api_call_start_time": None,
             "litellm_params": litellm_params,
         }
+        new_call.finalized_cost = None
+        new_call.prepared_cost = None
+        new_call.cost_allowance_discount = None
+        new_call.cost_finalization_lock = asyncio.Lock()
+        new_call.deferred_success_result = None
         new_call.streaming_chunks = []
         new_call.sync_streaming_chunks = []
         return new_call
@@ -2079,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
             elif router_model_id is None and "model_id" in hidden_params:  # use model_id if not already set
                 router_model_id = hidden_params["model_id"]
 
+        if isinstance(priced_result, LiteLLMBatch) and (
+            not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
+        ):
+            return 0.0
+
         # Fallback: extract router_model_id from litellm_params when not available
         # from the result object. ResponsesAPIResponse objects (used by /v1/responses
         # streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
@@ -2462,6 +2480,55 @@ class Logging(LiteLLMLoggingBaseClass):
             **kwargs,
         )
 
+    def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
+        if self.prepared_cost is not None:
+            return self.prepared_cost
+        recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
+        deferred: Final = self.deferred_success_result
+        prepared: Final = self.prepare_success_logging(
+            result=deferred[0] if deferred is not None else response,
+            start_time=deferred[1] if deferred is not None else self.start_time,
+            end_time=deferred[2] if deferred is not None else now.replace(tzinfo=None),
+            cache_hit=recorded_cache_hit if isinstance(recorded_cache_hit, bool) else None,
+            standard_logging_object=None,
+            build_logging_payload=True,
+            publish_logging_payload=False,
+        )
+        payload: Final = prepared[3]
+        raw_cost: Final[object] = self.model_call_details.get("response_cost")
+        if payload is None or not isinstance(raw_cost, (int, float)):
+            return None
+        recorded_usage: Final[object] = self.model_call_details.get("combined_usage_object")
+        response_usage: Final[object] = getattr(prepared[2], "usage", None)
+        normalized_usage: Final = (
+            recorded_usage
+            if isinstance(recorded_usage, Usage)
+            else response_usage
+            if isinstance(response_usage, Usage)
+            else None
+        )
+        usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(
+            _extract_response_obj_and_hidden_params(prepared[2], None)[0],
+            combined_usage_object=normalized_usage,
+        ).model_copy(deep=True)
+        self.prepared_cost = PreparedCost(
+            raw_cost,
+            {
+                **payload,
+                "response": None,
+                "messages": None,
+                "model_map_information": copy.deepcopy(payload["model_map_information"]),
+                "cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
+            },
+            usage,
+        )
+        self.record_partial_usage_for_failure(usage, raw_cost)
+        return self.prepared_cost
+
+    def record_finalized_cost(self, finalized: FinalizedCost) -> None:
+        self.finalized_cost = finalized
+        self.model_call_details["response_cost"] = finalized.amount
+
     def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
         """Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
         self.model_call_details["combined_usage_object"] = usage
@@ -2812,12 +2879,16 @@ class Logging(LiteLLMLoggingBaseClass):
                     logging_result, "_hidden_params", {}
                 )
 
-        if self.model_call_details.get("cache_hit") is True:
+        if self.finalized_cost is not None:
+            self.model_call_details["response_cost"] = self.finalized_cost.amount
+        elif self.model_call_details.get("cache_hit") is True:
             self.model_call_details["response_cost"] = 0.0
         elif "response_cost" in hidden_params:
             self.model_call_details["response_cost"] = hidden_params["response_cost"]
             self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
-        elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
+        elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
+            existing_cost != 0 or self.call_type == CallTypes.pass_through.value  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
+        ):
             # Preserve response_cost if already calculated (e.g., by pass-through
             # handlers like Gemini/Vertex which call completion_cost directly).
             # Do not preserve 0 from failure_handler on intermediate router retries.
@@ -2936,7 +3007,7 @@ class Logging(LiteLLMLoggingBaseClass):
         standard_logging_object: StandardLoggingPayload | None = None,
         build_logging_payload: bool = True,
     ) -> tuple[datetime.datetime, datetime.datetime, object]:
-        prepared: Final = self._prepare_success_logging(
+        prepared: Final = self.prepare_success_logging(
             result=result,
             start_time=start_time,
             end_time=end_time,
@@ -2947,7 +3018,7 @@ class Logging(LiteLLMLoggingBaseClass):
         )
         return prepared[0], prepared[1], prepared[2]
 
-    def _prepare_success_logging(
+    def prepare_success_logging(
         self,
         result: object,
         start_time: datetime.datetime | None,
@@ -2968,7 +3039,11 @@ class Logging(LiteLLMLoggingBaseClass):
 
             self.model_call_details["log_event_type"] = "successful_api_call"
             self.model_call_details["end_time"] = end_time
-            self.model_call_details["cache_hit"] = cache_hit
+            self.model_call_details["cache_hit"] = (
+                self.prepared_cost.payload["cache_hit"]
+                if cache_hit is None and self.prepared_cost is not None
+                else cache_hit
+            )
 
             if self.call_type == CallTypes.anthropic_messages.value:
                 result = self._anthropic_messages_logged_response(result=result)
@@ -3007,6 +3082,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             if (
                 litellm.max_budget
+                and publish_logging_payload
                 and self.stream is False
                 and result is not None
                 and isinstance(result, dict)
@@ -3037,10 +3113,17 @@ class Logging(LiteLLMLoggingBaseClass):
         publish_logging_payload: bool,
     ) -> StandardLoggingPayload | None:
         if standard_logging_object is not None:
+            effective_payload: Final = (
+                self.finalized_cost.logging_payload(standard_logging_object)
+                if self.finalized_cost is not None
+                else standard_logging_object
+            )
+            if self.finalized_cost is not None:
+                self.model_call_details["response_cost"] = self.finalized_cost.amount
             if not publish_logging_payload:
-                return standard_logging_object.copy()
-            self.model_call_details["standard_logging_object"] = standard_logging_object
-            return standard_logging_object
+                return effective_payload.copy()
+            self.model_call_details["standard_logging_object"] = effective_payload
+            return effective_payload
         if result is None or self.stream is True:
             # Streaming reaches here before its cost is known, so the cost
             # is seeded to None, but only when nothing has already
@@ -3269,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
         finally:
             self._restore_correlation_context()
 
+    def _is_batch_poll_without_logging(self, result: object) -> bool:
+        litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
+        return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
+
     def _success_handler_body(
         self,
         result: object = None,
@@ -3281,7 +3368,7 @@ class Logging(LiteLLMLoggingBaseClass):
         verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
         if not self.should_run_logging(event_type="sync_success"):  # prevent double logging
             return
-        start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
+        start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
             start_time=start_time,
             end_time=end_time,
             result=result,
@@ -3783,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## CALCULATE COST FOR BATCH JOBS
         if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
-            litellm_params = self.litellm_params or {}
-            litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
-            if (
-                litellm_metadata.get("batch_ignore_default_logging", False) is True
-            ):  # polling job will query these frequently, don't spam db logs
+            if self._is_batch_poll_without_logging(result):
                 return
 
             batch_cost: Final = kwargs.get("batch_cost", None)
@@ -4204,7 +4287,9 @@ class Logging(LiteLLMLoggingBaseClass):
         # chunks already delivered; the router stashes that recovered usage as
         # ``combined_usage_object`` and pre-computes its cost, so preserve it
         # here instead of zeroing the spend on an otherwise-failed request.
-        if self.model_call_details.get("combined_usage_object") is None:
+        if self.finalized_cost is not None:
+            self.model_call_details["response_cost"] = self.finalized_cost.amount
+        elif self.model_call_details.get("combined_usage_object") is None:
             self.model_call_details["response_cost"] = 0
 
         if hasattr(exception, "headers") and isinstance(exception.headers, dict):
@@ -5544,7 +5629,7 @@ def build_custom_logger_compatible_class(
                     return callback
             otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
             otel_logger_v2: Final = build_otel_v2_logger(
-                config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
+                config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
             )
             _in_memory_loggers.append(otel_logger_v2)
             _maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
@@ -7367,6 +7452,12 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
     )
 
 
+def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
+    """Duck-typed logging objects (and test doubles) carry no finalized cost."""
+    finalized: Final = getattr(logging_obj, "finalized_cost", None)
+    return finalized if isinstance(finalized, FinalizedCost) else None
+
+
 def get_standard_logging_object_payload(
     kwargs: dict | None,
     init_response_obj: object,
@@ -7396,7 +7487,8 @@ def get_standard_logging_object_payload(
         # Extract usage as a plain dict, avoiding Pydantic round-trip
         raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
             response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
-            combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
+            combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
+            or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
         )
         usage_dict: Final = (
             {**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
@@ -7470,7 +7562,10 @@ def get_standard_logging_object_payload(
         raw_response_cost: Final = kwargs.get("response_cost")
         llm_response_cost: Final[float] = raw_response_cost or 0.0
         guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
-        response_cost: Final[float] = llm_response_cost + guardrail_cost
+        finalized_cost: Final = _finalized_cost_of(logging_obj)
+        response_cost: Final[float] = (
+            finalized_cost.amount if finalized_cost is not None else llm_response_cost + guardrail_cost
+        )
 
         # clean up litellm hidden params
         clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
@@ -7537,13 +7632,17 @@ def get_standard_logging_object_payload(
         ):
             model_name = response_model_name
 
-        request_cost_breakdown: Final = cost_breakdown_with_guardrail(
-            (
-                _cache_hit_cost_breakdown(uncached_price.cost_breakdown)
-                if uncached_price is not None
-                else logging_obj.cost_breakdown
-            ),
-            guardrail_cost,
+        request_cost_breakdown: Final = (
+            finalized_cost.logging_breakdown()
+            if finalized_cost is not None
+            else cost_breakdown_with_guardrail(
+                (
+                    _cache_hit_cost_breakdown(uncached_price.cost_breakdown)
+                    if uncached_price is not None
+                    else logging_obj.cost_breakdown
+                ),
+                guardrail_cost,
+            )
         )
         captured_baseline: Final = logging_obj.baseline_observation
         autorouter_savings: Final = (
@@ -7600,6 +7699,7 @@ def get_standard_logging_object_payload(
             model=model_name,
             metadata=clean_metadata,
             cache_key=clean_hidden_params["cache_key"],
+            cost_finalized=finalized_cost is not None,
             response_cost=response_cost,
             cost_breakdown=request_cost_breakdown,
             autorouter_savings=autorouter_savings,

litellm/litellm_core_utils/llm_cost_calc/finalization.py

@@ -0,0 +1,61 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final, TypeAlias
+
+from pydantic import TypeAdapter
+
+from litellm.types.utils import CostBreakdown, StandardLoggingPayload, Usage
+
+BreakdownValue: TypeAlias = float | str | None | Mapping[str, float]
+_BREAKDOWN_FIELDS: Final = TypeAdapter(Mapping[str, BreakdownValue])
+_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
+
+
+@dataclass(frozen=True, slots=True)
+class PreparedCost:
+    amount: float
+    payload: StandardLoggingPayload
+    usage: Usage
+
+
+@dataclass(frozen=True, slots=True)
+class FinalizedCost:
+    amount: float
+    breakdown: Mapping[str, BreakdownValue] | None
+    allowance_discount: float
+
+    def logging_payload(self, payload: StandardLoggingPayload) -> StandardLoggingPayload:
+        return {
+            **payload,
+            "response_cost": self.amount,
+            "cost_breakdown": self.logging_breakdown(),
+            "cost_finalized": True,
+        }
+
+    def logging_breakdown(self) -> CostBreakdown | None:
+        return _COST_BREAKDOWN.validate_python(self.breakdown) if self.breakdown is not None else None
+
+
+def finalize_cost(amount: float, breakdown: CostBreakdown | None, allowance_discount: float = 0.0) -> FinalizedCost:
+    discount: Final = min(max(amount, 0.0), max(allowance_discount, 0.0))
+    total: Final = amount - discount
+    existing: Final = _BREAKDOWN_FIELDS.validate_python(breakdown) if breakdown is not None else None
+    adjusted: Final = (
+        {
+            **(existing or {}),
+            "total_cost": total,
+            "original_cost": (breakdown or {}).get("original_cost", amount),
+            "discount_amount": (breakdown or {}).get("discount_amount", 0.0) + discount,
+        }
+        if discount > 0
+        else existing
+    )
+    frozen: Final = (
+        MappingProxyType(
+            {key: MappingProxyType(value) if isinstance(value, Mapping) else value for key, value in adjusted.items()}
+        )
+        if adjusted is not None
+        else None
+    )
+    return FinalizedCost(amount=total, breakdown=frozen, allowance_discount=discount)