Merge remote-tracking branch 'origin/main' into litellm_stream_error_alert_debug_info

# Conflicts:
#	tests/unit/text_completions/test_streaming_iterator.py

.circleci/scripts/run_integration.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [ "${GITHUB_ACTIONS:-}" = true ]; then
-  echo "Integration contracts are owned by CircleCI" >&2
-  exit 1
-fi
-
 suite="${1:?integration suite required}"
 mode="${2:-standard}"
 side="${3:-}"
+requested_workers="${INTEGRATION_WORKERS:-}"
+license_env=()
+if [ -n "${LITELLM_LICENSE:-}" ]; then
+  license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
+fi
 if [ "$mode" = replica ]; then
   results="test-results/integration-${suite}-replica"
 elif [ "$mode" = parity ]; then
@@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
   export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
   test -f "$LITELLM_UI_PATH/index.html"
 fi
-export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
+export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
 export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
 
 uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@@ -107,7 +107,7 @@ sudo iptables -N integration_only
 guard_created=true
 sudo iptables -A integration_only -o lo -j ACCEPT
 sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
-for service in postgres-db redis-cache; do
+for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
   address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
   sudo iptables -A integration_only -d "$address" -j ACCEPT
 done
@@ -145,6 +145,9 @@ fi
 if [ "$suite" = mcp ]; then
   export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
 fi
+if [ -n "$requested_workers" ]; then
+  export INTEGRATION_WORKERS="$requested_workers"
+fi
 isolated_workers=false
 if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
   isolated_workers=true
@@ -185,7 +188,7 @@ start_proxy() {
     "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
-    LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+    "${license_env[@]}" \
     LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
     "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   "${shared_services[@]}" "${worker_coverage[@]}" \
   INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   LITELLM_MODE=PRODUCTION \
-  LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
+  "${license_env[@]}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
   INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
+/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
 /.github/workflows/zizmor.yml @yuneng-berri

.github/scripts/assert_secret_absent.py

@@ -0,0 +1,119 @@
+import base64
+import io
+import os
+import sys
+import urllib.parse
+import zipfile
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from functools import reduce
+from itertools import chain
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, TextIO
+
+SECRET_VARIABLE: Final = "SCANNED_SECRET"
+MIN_ENCODED_LENGTH: Final = 12
+MAX_ARCHIVE_DEPTH: Final = 3
+MASK: Final = "***"
+
+Forms = tuple[tuple[str, str], ...]
+
+
+@dataclass(frozen=True, slots=True)
+class Hit:
+    location: str
+    form: str
+
+
+def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
+    for offset in range(3):
+        chunk: Final = raw[offset:]
+        yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
+
+
+def secret_forms(secret: str) -> Forms:
+    encoded: Final = (
+        ("url-encoded", urllib.parse.quote(secret, safe="")),
+        *_aligned_encodings(secret.encode(), "base64", base64.b64encode),
+        *_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
+    )
+    by_value: Final = MappingProxyType(
+        {value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
+    )
+    return (("raw", secret), *((name, value) for value, name in by_value.items()))
+
+
+def mask(text: str, forms: Forms) -> str:
+    return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
+
+
+def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
+    for form, value in forms:
+        if value in name:
+            yield Hit(mask(location, forms), f"{form} in name")
+
+
+def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
+    for form, value in forms:
+        if value.encode() in data:
+            yield Hit(mask(location, forms), form)
+    if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
+        return
+    with zipfile.ZipFile(io.BytesIO(data)) as archive:
+        for member in archive.infolist():
+            if member.is_dir():
+                continue
+            yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
+
+
+def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
+    for directory, subdirectories, files in os.walk(root):
+        yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
+        yield from ((Path(directory, name), False) for name in files)
+
+
+def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
+    location: Final = str(path.relative_to(root))
+    if linked_directory:
+        yield Hit(mask(location, forms), "directory symlink, not scanned")
+        return
+    yield from _name_hits(location, location, forms)
+    yield from _content_hits(location, path.read_bytes(), forms, 0)
+
+
+def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
+    forms: Final = secret_forms(secret)
+    entries: Final = tuple(_walk(root))
+    hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
+    return len(entries), hits
+
+
+def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
+    if len(argv) != 1:
+        print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
+        return 2
+    secret: Final = environ.get(SECRET_VARIABLE, "")
+    root: Final = Path(argv[0])
+    if not secret:
+        print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
+        return 0
+    if not root.is_dir():
+        print(f"{root} is not a directory, nothing to scan", file=out)
+        return 0
+    try:
+        scanned, hits = scan(root, secret)
+    except Exception as error:
+        print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
+        return 1
+    for hit in hits:
+        print(f"secret found: {hit.location} ({hit.form})", file=out)
+    if hits:
+        print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
+        return 1
+    print(f"no copy of the secret in {scanned} files under {root}", file=out)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

.github/scripts/smoke_test_native_wheel.py

@@ -7,6 +7,8 @@ import zipfile
 from pathlib import Path
 from typing import Final
 
+COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json"
+
 CHILD_SCRIPT: Final = """
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
@@ -34,6 +36,9 @@ else:
 after = module.gil_stats()
 if not isinstance(after.get("releases"), int):
     raise AssertionError(f"native module unusable after panic: {after!r}")
+
+if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes():
+    raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout")
 """
 
 
@@ -55,7 +60,9 @@ def main() -> int:
 
         native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
         native_path.write_bytes(archive.read(native_members[0]))
-        result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
+        result: Final = subprocess.run(
+            (sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False
+        )
 
     if result.returncode != 0:
         sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")

.github/workflows/cost-map-guard.yml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@@ -21,16 +22,21 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head and its merge base
+      - name: Check out the base branch tip and fetch the pull request head and its merge base
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
+          git checkout -q --detach "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv
         uses: ./.github/actions/setup-uv-with-retries
         with:
@@ -38,7 +44,30 @@ jobs:
       - name: Run the guard
         env:
           MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
+          COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |
           uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
+      - name: Check whether the pull request changes the cost map
+        id: changed
+        env:
+          MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
+            echo "cost_map=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "cost_map=true" >> "$GITHUB_OUTPUT"
+          fi
+      - if: steps.changed.outputs.cost_map == 'true'
+        run: rustup toolchain install --no-self-update
+      - name: Load the cost map through model-catalog
+        if: steps.changed.outputs.cost_map == 'true'
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
+          cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
+            -- "$RUNNER_TEMP/model_prices_and_context_window.json"

.github/workflows/cost-map-pins-recheck.yml

@@ -0,0 +1,91 @@
+name: Cost map pins recheck
+
+on:
+  push:
+    branches:
+      - main
+      - "litellm_**"
+    paths:
+      - model_prices_and_context_window.json
+      - litellm/model_prices_and_context_window_backup.json
+      - ci_cd/cost_map_pins.json
+
+permissions: {}
+
+jobs:
+  rerun-cost-map-guard:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Re-run the cost map guard on open pull requests the push could break
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ github.ref_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
+          FORCED: ${{ github.event.forced }}
+          COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
+          PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
+        run: |
+          if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
+            || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
+              --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
+              | grep -qx true; then
+            export RECHECK_PATHS="$COST_MAP_PATHS"
+          else
+            export RECHECK_PATHS="$PINS_PATHS"
+          fi
+          rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
+          candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
+            -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
+                  nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
+                  pageInfo { hasNextPage endCursor }
+                }
+              }
+            }' \
+            --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
+              | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
+              | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
+          failed=0
+          declare -A rerun_heads
+          while read -r number head_sha listed; do
+            [ -n "$number" ] || continue
+            if [ "$listed" = false ]; then
+              if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
+                --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
+                echo "::error::could not list the files of #$number"
+                failed=1
+                continue
+              fi
+              [ -n "$changed" ] || continue
+            fi
+            [ -z "${rerun_heads[$head_sha]:-}" ] || continue
+            rerun_heads[$head_sha]=1
+            if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
+              echo "::error::could not list the cost map guard runs of #$number"
+              failed=1
+              continue
+            fi
+            if [ -z "$runs" ]; then
+              echo "::warning::no cost map guard run found for #$number"
+              continue
+            fi
+            while read -r run_id created_at; do
+              if [[ "$created_at" < "$rerun_cutoff" ]]; then
+                echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
+                continue
+              fi
+              gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
+              if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
+                echo "::error::could not re-run cost map guard run $run_id for #$number"
+                failed=1
+              fi
+            done <<< "$runs"
+          done <<< "$candidates"
+          exit "$failed"

.github/workflows/image-scan.yml

@@ -13,8 +13,10 @@ on:
       - migrations/Dockerfile
       - migrations/run.py
       - gateway/Dockerfile
+      - gateway/launch.py
       - gateway/main.py
       - gateway/routes/allowlist.py
+      - litellm_cli/**
       - backend/Dockerfile
       - backend/main.py
       - litellm-rust/**

.github/workflows/lens-install-smoke.yml

@@ -1,152 +0,0 @@
-name: Lens installation smoke
-
-on:
-  workflow_dispatch:
-    inputs:
-      lens_image:
-        description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      lens_upgrade_image:
-        description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
-        required: true
-        type: string
-      gateway_baseline_ref:
-        description: Previously qualified gateway source commit, full SHA
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: lens-install-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-images:
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-          - component: backend
-            dockerfile: backend/Dockerfile
-          - component: ui
-            dockerfile: ui/Dockerfile
-          - component: migrations
-            dockerfile: migrations/Dockerfile
-          - component: monolith
-            dockerfile: Dockerfile
-          - component: gateway
-            dockerfile: gateway/Dockerfile
-            baseline: true
-          - component: backend
-            dockerfile: backend/Dockerfile
-            baseline: true
-          - component: monolith
-            dockerfile: Dockerfile
-            baseline: true
-    env:
-      COMPONENT: ${{ matrix.component }}
-      DOCKERFILE: ${{ matrix.dockerfile }}
-      SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
-      IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
-      CURRENT_REF: ${{ github.sha }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ env.SOURCE_REF }}
-      - name: Verify the selected source
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          test "$(git rev-parse HEAD)" = "$SOURCE_REF"
-          if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
-            test "$SOURCE_REF" != "$CURRENT_REF"
-          fi
-      - name: Build the matching release image
-        run: |
-          docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-            -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
-      - name: Save the matching release image
-        run: |
-          docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
-            | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
-          path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
-          compression-level: 0
-          retention-days: 3
-          if-no-files-found: error
-          overwrite: true
-
-  helm-install:
-    needs: build-images
-    runs-on: ubuntu-latest-16-cores
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          pattern: lens-install-*-${{ github.sha }}
-          merge-multiple: true
-          path: ${{ runner.temp }}/lens-install-images
-      - name: Load the matching release images
-        run: |
-          for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
-            gzip -dc "$archive" | docker load
-            rm "$archive"
-          done
-      - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
-        with:
-          cosign-release: v2.4.3
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ghcr.io
-          username: ${{ vars.LENS_READ_USER || github.actor }}
-          password: ${{ secrets.LENS_READ_TOKEN }}
-      - name: Verify independent Lens images before loading them
-        env:
-          LENS_IMAGE: ${{ inputs.lens_image }}
-          LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
-          LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
-          LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
-        shell: bash -euo pipefail {0}
-        run: |
-          [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
-          printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
-          echo "$LENS_PUBLIC_KEY_SHA256  $RUNNER_TEMP/lens.pub" | sha256sum --check
-          for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
-            [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
-            cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
-            cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
-            docker pull "$image"
-          done
-          docker tag "$LENS_IMAGE" lens-ci-worker:baseline
-          docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
-      - name: Install pinned Kubernetes test tools
-        run: |
-          curl --fail --location --output "$RUNNER_TEMP/kind" \
-            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
-          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kind"
-          curl --fail --location --output "$RUNNER_TEMP/kubectl" \
-            https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
-          echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea  $RUNNER_TEMP/kubectl" | sha256sum --check
-          chmod +x "$RUNNER_TEMP/kubectl"
-          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
-            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
-          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
-          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
-          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
-          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
-      - name: Install, ingest, upgrade, and restart both charts
-        run: bash tests/e2e/migrations/lens_helm_smoke.sh

.github/workflows/spend-logs-guard.yml

@@ -19,17 +19,17 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          ref: ${{ github.event.pull_request.base.sha }}
+          ref: ${{ github.event.pull_request.base.ref }}
           persist-credentials: false
       - name: Fetch the pull request head, its merge base and its reviews
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

.github/workflows/test-integration.yml

@@ -0,0 +1,199 @@
+name: integration
+
+on:
+  pull_request:
+    types: [labeled]
+  schedule:
+    - cron: "5 8-20/3 * * 1-6"
+      timezone: America/Los_Angeles
+  workflow_dispatch:
+    inputs:
+      suite:
+        description: Integration group to run
+        type: choice
+        options:
+          - all
+          - providers
+          - extensions
+          - accounting
+          - management
+          - management-replica
+          - database
+          - database-replica
+          - mcp
+          - security
+          - cost
+          - sdk
+          - browser
+        default: all
+      workers:
+        description: Workers per group, blank keeps each group's default
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  integration:
+    name: integration ${{ matrix.suite }}
+    if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
+    environment: integration-license
+    runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
+    env:
+      UV_PYTHON: "3.12"
+      INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: circle_test
+        ports:
+          - 5432:5432
+        options: >-
+          --shm-size 1g
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+      redis:
+        image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Cache Prisma binaries
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Install redis-server 7.2.16
+        timeout-minutes: 3
+        run: |
+          mkdir -p "$RUNNER_TEMP/redis-bin"
+          cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
+          docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
+          docker rm "$cid"
+          "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
+          echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
+
+      - name: Install MCP SDK1 peer
+        if: matrix.suite == 'mcp'
+        timeout-minutes: 5
+        run: |
+          uv venv --python 3.12 .venv-mcp-peer
+          uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
+          echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
+
+      - name: Set up Node.js
+        if: matrix.suite == 'browser'
+        timeout-minutes: 5
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version-file: ui/litellm-dashboard/.nvmrc
+          cache: "npm"
+          cache-dependency-path: |
+            ui/litellm-dashboard/package-lock.json
+            tests/e2e/ui/package-lock.json
+
+      - name: Install browser dependencies and build the dashboard
+        if: matrix.suite == 'browser'
+        timeout-minutes: 20
+        run: |
+          (cd ui/litellm-dashboard && npm ci)
+          (cd tests/e2e/ui && npm ci)
+          sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
+            node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
+          node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
+          (cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
+
+      - name: Run integration contracts
+        timeout-minutes: 45
+        env:
+          SUITE: ${{ matrix.suite }}
+          INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
+          INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
+          LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
+        run: |
+          case "$SUITE" in
+            *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
+            *) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
+          esac
+
+      - name: Verify every case ran exactly once
+        if: ${{ !cancelled() && matrix.suite != 'browser' }}
+        timeout-minutes: 2
+        run: |
+          record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
+          test -n "$record"
+          mkdir -p "$RUNNER_TEMP/shards/0"
+          cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
+          .venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
+
+      - name: Report MCP coverage
+        if: ${{ !cancelled() && matrix.suite == 'mcp' }}
+        timeout-minutes: 5
+        run: |
+          coverage_dir=test-results/integration-mcp/coverage
+          test -f "$coverage_dir/coverage.txt" && exit 0
+          export COVERAGE_FILE="$PWD/$coverage_dir/data"
+          .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
+          .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
+          .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
+          tail -n 1 "$coverage_dir/coverage.txt"
+
+      - name: Check results for the license
+        id: secret-scan
+        if: ${{ !cancelled() }}
+        timeout-minutes: 5
+        env:
+          SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
+        run: python .github/scripts/assert_secret_absent.py test-results
+
+      - name: Upload results
+        if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
+        timeout-minutes: 10
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: test-results/
+          retention-days: 14

.github/workflows/test-linting.yml

@@ -48,11 +48,13 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$MERGE_BASE"
           retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
           echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@@ -264,6 +266,7 @@ jobs:
     name: ui
     permissions:
       contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     timeout-minutes: 8
     defaults:
@@ -281,16 +284,12 @@ jobs:
         id: changed
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          # base.sha is the base branch tip from when the PR was opened, while
-          # actions/checkout leaves HEAD on a merge of the PR into the *current*
-          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
-          # landed since, so a PR that touches no UI file still gets linted
-          # against hundreds of other people's files. Diff the PR head against its
-          # own merge base instead, which is exactly what this PR changed.
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"
@@ -376,10 +375,10 @@ jobs:
             exit 0
           fi
           files="$(git diff --name-only "$base" HEAD)"
-          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
+          if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
-            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
             echo "relevant=false" >> "$GITHUB_OUTPUT"
           fi
 
@@ -630,6 +629,12 @@ jobs:
       - name: check_migrations_lock_timeout
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
 
+      - name: check_no_none_string_error_fields
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
+
+      - name: test_check_no_none_string_error_fields
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
+
       - name: check_no_publicly_known_master_key
         run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
 

.github/workflows/test-terraform-provider.yml

@@ -12,6 +12,7 @@ on:
     paths:
       - "terraform/provider/**"
       - "litellm/proxy/**"
+      - "enterprise/**"
       - ".github/workflows/test-terraform-provider.yml"
 
 permissions:

.github/workflows/test-unit.yml

@@ -227,6 +227,7 @@ jobs:
               tests/unit/llms/test_oss_decision.py
               tests/unit/llms/test_polling_url_origin_match.py
               tests/unit/llms/test_predibase_transformation.py
+              tests/unit/llms/test_subscription_accounts.py
               tests/unit/llms/a2a
               tests/unit/llms/aiml
               tests/unit/llms/aiohttp_openai
@@ -236,6 +237,7 @@ jobs:
               tests/unit/llms/azure_ai
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
+              tests/unit/llms/bigmodel
               tests/unit/llms/black_forest_labs
               tests/unit/llms/brave
               tests/unit/llms/bytez
@@ -244,6 +246,7 @@ jobs:
               tests/unit/llms/chatgpt
               tests/unit/llms/clarifai
               tests/unit/llms/claude_code
+              tests/unit/llms/clf_ai_gateway
               tests/unit/llms/cloudflare
               tests/unit/llms/codestral
               tests/unit/llms/codex
@@ -321,6 +324,7 @@ jobs:
               tests/unit/llms/openai_like
               tests/unit/llms/opencode
               tests/unit/llms/opensandbox
+              tests/unit/llms/opensearch
               tests/unit/llms/openrouter
               tests/unit/llms/ovhcloud
               tests/unit/llms/parallel_ai
@@ -1001,9 +1005,9 @@ jobs:
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1148,18 +1152,20 @@ jobs:
         env:
           CI: "true"
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
 
-          if [ -z "$BASE_SHA" ]; then
+          if [ -z "$PR_NUMBER" ]; then
             echo "Push to $GITHUB_REF_NAME: running the full suite"
             full_suite
             exit 0
           fi
 
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$merge_base"
           git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           changed_files=()
@@ -1449,9 +1455,9 @@ jobs:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
         run: |
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
           fi

Makefile

@@ -153,7 +153,7 @@ lint-install:
 # the same set CI's ':(glob)litellm/**/*.py' selects.
 lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	@base_ref=$$($(RESOLVE_BASE)) && \
-	changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
+	changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
 	files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
 	if [ -z "$$files" ]; then \
 		echo "No changed litellm Python files to format-check."; \

README.md

@@ -141,12 +141,56 @@ response = client.chat.completions.create(
 )
 ```
 
-[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
+</details>
+
+<details>
+<summary><b>Auto Routing</b> - Choose a model for each prompt (Python SDK + AI Gateway)</summary>
+
+### Python SDK
+
+[**Docs: Autorouter SDK**](https://docs.litellm.ai/docs/autorouter_sdk)
+
+Install with `uv add litellm` and set `OPENAI_API_KEY` for the models below.
+
+```python
+import asyncio
+from litellm import Router
+
+router = Router(model_list=[
+    {"model_name": "fast", "litellm_params": {"model": "openai/gpt-6-luna"}},
+    {"model_name": "capable", "litellm_params": {"model": "openai/gpt-6.1-sol"}},
+    {"model_name": "auto", "litellm_params": {
+        "model": "auto_router/complexity_router",
+        "complexity_router_config": {
+            "classifier_type": "heuristic",
+            "tiers": {"SIMPLE": "fast", "MEDIUM": "fast",
+                      "COMPLEX": "capable", "REASONING": "capable"},
+            "return_raw_model_name": True,
+        },
+        "complexity_router_default_model": "capable",
+    }},
+])
+response = asyncio.run(router.acompletion(
+    model="auto", messages=[{"role": "user", "content": "What is 2+2?"}],
+))
+print(response.model, response.choices[0].message.content)
+```
+
+### AI Gateway (Proxy Server)
+
+[**Create an Auto Router**](https://docs.litellm.ai/docs/auto_router/setup), then call its model name. Replace `auto` and the URL below with yours, and set `LITELLM_API_KEY` to your virtual key.
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"auto","messages":[{"role":"user","content":"What is 2+2?"}]}'
+```
 
 </details>
 
 <details>
-<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
+<summary><b>A2A</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
 
 [**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
 
@@ -335,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -354,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) |  |  |  |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -442,7 +488,7 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
 <tr>
 <th style={{width: '14%'}}></th>
 <th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
-<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/python_sdk">LiteLLM Python SDK</a></strong></th>
 </tr>
 </thead>
 <tbody>

backend/Dockerfile

@@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0
 COPY pyproject.toml README.md rust-toolchain.toml ./
 COPY .cargo ./.cargo
 COPY litellm-rust ./litellm-rust
+COPY model_prices_and_context_window.json ./
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     mkdir -p litellm/rust_bridge /wheels && \

backend/routes/allowlist.py

@@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/subscription_accounts/",
     "/cost_optimization/",
     "/cost/",
     "/credentials",

ci_cd/cost_map_guard.py

@@ -1,37 +1,67 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-get the file checks and additionally may only touch those two files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
+file checks: the files parse, the backup copy matches the root file, and every capability pinned in
+ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
+otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
+a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
+all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
+whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
+only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
+(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
+Bot) get the human checks and may not touch the pins file either.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import subprocess
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+PINS_PATH: Final = "ci_cd/cost_map_pins.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
+CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
+TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
+AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
+BOT_AUTHOR_TYPE: Final = "Bot"
+MERGE_CONFLICT_EXIT_CODE: Final = 1
 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
+_MISSING: Final = object()
 
 
 @dataclass(frozen=True, slots=True)
 class Snapshot:
     cost_map: str
     backup: str
+    pins: str
+
+
+@dataclass(frozen=True, slots=True)
+class MergeFailed:
+    stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    key: str
+    field: str
+    value: object
+    checked: str
+    source: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
     return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
 
 
+def _pin(key: str, field: str, spec: object) -> Pin | str:
+    malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
+    if not isinstance(spec, dict) or "value" not in spec:
+        return malformed
+    checked: Final = spec.get("checked")
+    source: Final = spec.get("source")
+    if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
+        return malformed
+    return Pin(key, field, spec["value"], checked, source)
+
+
+def _pins(parsed: CostMap) -> Iterator[Pin | str]:
+    for key, fields in parsed.items():
+        if not isinstance(fields, dict):
+            yield f"{PINS_PATH}: {key} must map field names to pins"
+            continue
+        for field, spec in fields.items():
+            yield _pin(key, field, spec)
+
+
+def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
+    for pin in _pins(parsed):
+        if isinstance(pin, str):
+            yield pin
+        elif pin.key not in entries:
+            yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
+        elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
+            yield (
+                f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
+                f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
+            )
+
+
+def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
+    if not pins_text:
+        return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
+    parsed: Final = _parse_object(pins_text, PINS_PATH)
+    if isinstance(parsed, str):
+        return (parsed,)
+    return tuple(_pin_violations(parsed, _entries(head_map)))
+
+
 def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
     base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
     if isinstance(base_map, str):
@@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
+    if PINS_PATH not in changed_files:
+        return ()
+    if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
+        return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    if author_type == BOT_AUTHOR_TYPE:
+        return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    return ()
+
+
 def touches_cost_map(changed_files: Sequence[str]) -> bool:
-    return any(path in GUARDED_PATHS for path in changed_files)
+    return any(path in CHECKED_PATHS for path in changed_files)
 
 
 def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
+def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
+    if merged is None:
+        return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
+    if not merged.pins and not base.pins:
+        return ()
+    merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
+    if isinstance(merged_map, str):
+        return (f"after merging into the base branch, {merged_map}",)
+    return pin_failures(merged.pins, merged_map)
+
+
 def guard_failures(
     base: Snapshot,
     head: Snapshot,
+    merged: Snapshot | None,
     changed_files: Sequence[str],
     bot: bool,
     *,
@@ -147,6 +241,7 @@ def guard_failures(
         *_file_failures(
             head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
         ),
+        *_merged_pin_failures(base, merged),
         *(_bot_failures(base, head_map, changed_files) if bot else ()),
     )
 
@@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
+
+
+def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
+    result: Final = subprocess.run(
+        ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    tree: Final = result.stdout.split("\n", 1)[0]
+    if result.returncode == 0:
+        return tree
+    if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
+        return None
+    return MergeFailed(result.stderr.strip())
 
 
 def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
     parser: Final = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base", required=True, help="merge base of the pull request")
     parser.add_argument("--head", required=True, help="head commit of the pull request")
+    env_target: Final = os.environ.get(TARGET_ENV)
+    parser.add_argument(
+        "--target",
+        default=env_target,
+        required=env_target is None,
+        help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
+    )
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
     if changed is None:
         print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
         return 1
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    if not bot and not touches_cost_map(changed):
+        print(f"cost map guard passed ({contract_for(bot, changed)})")
+        return 0
+    merged: Final = merged_tree(args.base, args.target, args.head)
+    if isinstance(merged, MergeFailed):
+        print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
+        return 1
+    failures: Final = (
+        *guard_failures(
+            snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
+        ),
+        *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
+    )
     contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")

ci_cd/cost_map_pins.json

@@ -0,0 +1,16 @@
+{
+    "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    },
+    "fireworks_ai/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    }
+}

codecov.yaml

@@ -15,9 +15,10 @@ ignore:
 flag_management:
   default_rules:
     carryforward: true
-  # Dead flags no CI job uploads anymore: their carried-forward sessions were
-  # measured against old revisions, and the stale line maps mark comment lines
-  # of since-edited files as missed, sinking patch coverage on unrelated PRs.
+  # Dead flags no CI job uploads anymore, and flags whose job is path-gated on
+  # PRs: their carried-forward sessions were measured against old revisions, and
+  # the stale line maps mark comment lines of since-edited files as missed,
+  # sinking patch coverage on unrelated PRs.
   individual_flags:
     - name: proxy-mgmt-behavior
       carryforward: false
@@ -99,6 +100,10 @@ flag_management:
       carryforward: false
     - name: proxy-db-endpoints-and-responses
       carryforward: false
+    - name: roi-postgres
+      carryforward: false
+    - name: redis-compat
+      carryforward: false
 
 component_management:
   individual_components:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -398,7 +398,7 @@ class CheckBatchCost:
         deployment. Once that deployment leaves the router, default fallbacks can
         silently send the retrieve to a provider that never saw the batch, so its
         404 must not retire a potentially recoverable row."""
-        return self.llm_router.get_deployment(model_id=model_id) is not None
+        return self.llm_router.registry().deployment(model_id) is not None
 
     @staticmethod
     def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
@@ -564,7 +564,7 @@ class CheckBatchCost:
     def _get_deployment_id_for_bare_model(
         self, bare_model_name: str, llm_provider: str
     ) -> Optional[str]:
-        model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
+        model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
         deployment_id = (
             self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
         )
@@ -580,7 +580,7 @@ class CheckBatchCost:
     ) -> Optional[str]:
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
-        for deployment in self.llm_router.get_model_list(model_name=None) or []:
+        for deployment in self.llm_router.registry().deployments() or []:
             litellm_params = deployment.get("litellm_params") or {}
             actual_model = litellm_params.get("model")
             if not isinstance(actual_model, str):
@@ -623,8 +623,9 @@ class CheckBatchCost:
         """
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
-        for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
-            deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
+        registry = self.llm_router.registry()
+        for deployment_id in registry.names(model_group, kind="ids"):
+            deployment_info = registry.deployment(deployment_id)
             if deployment_info is None:
                 continue
             try:
@@ -743,7 +744,7 @@ class CheckBatchCost:
             except (IndexError, AttributeError):
                 pass
 
-        credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+        credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
         _file_content = await afile_content(
             file_id=cast(str, raw_output_file_id),
             _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
@@ -766,7 +767,7 @@ class CheckBatchCost:
             except Exception:
                 pass
 
-        deployment_info = self.llm_router.get_deployment(model_id=model_id)
+        deployment_info = self.llm_router.registry().deployment(model_id)
         if deployment_info is None:
             verbose_proxy_logger.info(
                 f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
@@ -996,7 +997,7 @@ class CheckBatchCost:
         )
 
         try:
-            deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
+            deployment_info: Final = self.llm_router.registry().deployment(model_id)
             if deployment_info is None:
                 verbose_proxy_logger.error(
                     f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
@@ -1009,7 +1010,7 @@ class CheckBatchCost:
                 custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
             )
             cost_provider: Final = cast(BatchFileProvider, llm_provider)
-            credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+            credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
             file_access: Final = {
                 **credentials,
                 "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -273,7 +273,7 @@ class CheckResponsesCost:
 
     def _resolve_deployment(self, response_id: str) -> Deployment | None:
         model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
-        return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
+        return self.llm_router.registry().deployment(model_id) if model_id is not None else None
 
     async def _get_response(
         self,
@@ -290,12 +290,14 @@ class CheckResponsesCost:
         sees provider env vars, so it fails for every deployment whose credentials
         live in the config; the row then never leaves ``queued``.
         """
-        if not via_router:
-            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
-        router_response = await self.llm_router.aget_responses(
-            response_id=response_id, litellm_metadata=litellm_metadata
+        retrieved: Final[object] = (
+            await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+            if via_router
+            else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
         )
-        return cast(ResponsesAPIResponse, router_response)
+        if not isinstance(retrieved, ResponsesAPIResponse):
+            raise TypeError(f"Retrieving {response_id} did not return a response object")
+        return retrieved
 
     async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
         return await self._get_response(
@@ -413,7 +415,7 @@ class CheckResponsesCost:
         if target.deployment is None or not isinstance(requested, str):
             return MappingProxyType({})
         routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
-            self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
+            self.llm_router.registry().deployments(requested, team_id=team_id) or ()
         )
         serving_id: Final = target.deployment.model_info.id
         return MappingProxyType(

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
     if llm_router is None or model_id is None:
         return None
     try:
-        credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
+        credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
     except Exception as error:
         verbose_logger.warning(
             "Failed to retrieve credentials for provider file "
@@ -2650,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
         }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
+            credentials = llm_router.registry().credentials(model_id, form="with_provider")
             delete_data = {
                 **filtered_data,
                 **(
@@ -2694,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     # against the deployment's configured bucket, which they only
                     # trust from this immutable server-side snapshot, never from
                     # request params.
-                    credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
+                    credentials = llm_router.registry().credentials(model_id, form="with_provider")
                     if credentials is not None:
                         data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
                     else:

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -290,7 +290,7 @@ def _check_team_project_limits(
 
 
 def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
-    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+    return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
 
 
 def _project_models_expanding_at_request_time(

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.80"
+version = "0.1.81"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.80"
+version = "0.1.81"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

examples/bigmodel.md

@@ -0,0 +1,63 @@
+# BigModel (Zhipu AI)
+
+Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
+
+```yaml
+model_list:
+  - model_name: glm
+    litellm_params:
+      model: bigmodel/glm-5.3
+      api_key: os.environ/BIGMODEL_API_KEY
+      api_base: https://open.bigmodel.cn
+```
+
+One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
+
+| Proxy endpoint | BigModel endpoint |
+| --- | --- |
+| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
+| `/v1/responses` | `/api/v1/responses` |
+| `/v1/messages` | `/api/anthropic/v1/messages` |
+
+`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
+
+A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
+
+The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
+
+```python
+import litellm
+
+chat = litellm.completion(
+    model="bigmodel/glm-5.3",
+    messages=[{"role": "user", "content": "Hello"}],
+    thinking={"type": "enabled"},
+    reasoning_effort="low",
+)
+
+response = litellm.responses(
+    model="bigmodel/glm-5.3",
+    input="Hello",
+    reasoning={"effort": "low"},
+    store=True,
+)
+
+async def messages():
+    return await litellm.anthropic.messages.acreate(
+        model="bigmodel/glm-5.3",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": "Hello"}],
+        thinking={"type": "adaptive"},
+        output_config={"effort": "low"},
+    )
+```
+
+Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
+
+GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
+
+Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
+
+The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
+
+API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

gateway/Dockerfile

@@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0
 COPY pyproject.toml README.md rust-toolchain.toml ./
 COPY .cargo ./.cargo
 COPY litellm-rust ./litellm-rust
+COPY model_prices_and_context_window.json ./
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     mkdir -p litellm/rust_bridge /wheels && \
@@ -144,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
 COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
 COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
 COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
+COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
 COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
 COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma

gateway/launch.py

@@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
 wins under token auth too, so exporting it here is enough for every worker to
 pick the pooled URL up unchanged.
 
+LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
+before it can answer its first 5 s health ping, so a slow host would kill each one.
+Workers then load the gateway app through ``worker_app`` without garbage collection
+pauses, which hold the GIL long enough to miss a ping.
+
 Run with:
     python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
 """
@@ -15,19 +20,19 @@ Run with:
 import os
 import sys
 from collections.abc import Callable, Mapping, Sequence
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from uvicorn.main import main as uvicorn_main
 
-from litellm.proxy.db.db_url_settings import DatabaseURLSettings
-from litellm.proxy.db.pgbouncer import (
-    PgBouncerError,
-    PgBouncerSettings,
-    export_pooled_database_url,
-    start_in_container_pgbouncer,
-)
+from litellm_cli.main import without_gc_pauses
 
-GATEWAY_APP: Final = "gateway.main:app"
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+    from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+    from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
+
+GATEWAY_APP: Final = "gateway.launch:worker_app"
 KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
 
 
@@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
     """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
     keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
     if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
-        return (GATEWAY_APP, *argv)
-    return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
+        return (GATEWAY_APP, "--factory", *argv)
+    return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
 
 
 def pool_database_url(
-    settings: DatabaseURLSettings,
-    pgbouncer: PgBouncerSettings,
+    settings: "DatabaseURLSettings",
+    pgbouncer: "PgBouncerSettings",
     environ: Mapping[str, str],
-) -> str | PgBouncerError | None:
+) -> "str | PgBouncerError | None":
     """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
 
     The upstream URL is whatever ``apply_to_env`` assembled from the discrete
     ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
     auth the pooler mints and renews the upstream token itself.
     """
+    from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
+
     if not pgbouncer.enabled:
         return None
     upstream_url: Final = environ.get("DATABASE_URL")
@@ -58,11 +65,20 @@ def pool_database_url(
     return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
 
 
+def worker_app() -> "FastAPI":
+    with without_gc_pauses():
+        from gateway.main import app
+    return app
+
+
 def _serve(argv: Sequence[str]) -> None:
     uvicorn_main(tuple(argv), prog_name="uvicorn")
 
 
 def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
+    from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+    from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
+
     settings: Final = DatabaseURLSettings.from_env()
     settings.apply_to_env()
     pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)

gateway/routes/allowlist.py

@@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
         "/test",
         "/debug/memory/summary",
         "/api/event_logging/batch",
+        "/eu_jev/v1/systemone",
     }
 )
 

helm/litellm-helm/Chart.lock

@@ -5,8 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
-generated: '2026-10-09T04:18:10.643228000Z'
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: '2026-10-30T02:13:11.505726000Z'

helm/litellm-helm/Chart.yaml

@@ -39,6 +39,3 @@ dependencies:
     version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm-helm/README.md

@@ -256,6 +256,10 @@ itself.
 
 ## Connect Lens
 
-`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+## Shutdown accounting
+
+`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
+
+Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`

helm/litellm-helm/templates/_helpers.tpl

@@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
   value: {{ .Values.collector.drainTimeoutSeconds | quote }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -402,95 +398,28 @@ shutdown drain window.
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.labels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
-{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm-helm/templates/ingress.yaml

@@ -1,8 +1,6 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "litellm.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- $lensMode := include "litellm.lens.mode" . -}}
-{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@@ -46,18 +44,18 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
+          {{- if and $.Values.lens.enabled $.Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
-                name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
+                name: {{ $.Values.lens.service.name }}
                 port:
-                  number: {{ $.Values.lensWorker.service.port }}
+                  number: {{ $.Values.lens.service.port }}
               {{- else }}
-              serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
-              servicePort: {{ $.Values.lensWorker.service.port }}
+              serviceName: {{ $.Values.lens.service.name }}
+              servicePort: {{ $.Values.lens.service.port }}
               {{- end }}
           {{- end }}
           {{- range .paths }}

helm/litellm-helm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm-helm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm-helm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm-helm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm-helm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm-helm/tests/lens_connection_tests.yaml

@@ -0,0 +1,128 @@
+suite: lens connection
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - ingress.yaml
+tests:
+  - it: connects the proxy to an external Lens
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service ahead of the host paths
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[0].path
+          value: /lens-ingest
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,84 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - deployment.yaml
-  - configmap-litellm.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest

helm/litellm-helm/tests/lens_modes_tests.yaml

@@ -1,187 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- ingress.yaml
-- configmap-litellm.yaml
-- deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm-helm/tests/lens_modes_validation_tests.yaml

@@ -1,52 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm-helm/tests/lens_saved_secrets_tests.yaml

@@ -1,59 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

helm/litellm-helm/tests/lens_service_tests.yaml

@@ -1,167 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- configmap-litellm.yaml
-- deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects deployment.yaml to the shared Lens service
-  template: deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.hosts:
-    - host: gateway.example
-      paths:
-      - path: /
-        pathType: Prefix
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in deployment.yaml
-  template: deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference

helm/litellm-helm/tests/lens_setup_tests.yaml

@@ -1,162 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60

helm/litellm-helm/values.yaml

@@ -668,70 +668,15 @@ serviceMonitor:
     # - test-namespace
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

helm/litellm/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: lens
-  repository: oci://ghcr.io/berriai/charts
-  version: 0.1.0-dev.0
-digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
-generated: '2026-10-09T04:18:10.642706000Z'

helm/litellm/Chart.yaml

@@ -6,8 +6,3 @@ version: 0.1.0
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
-
-dependencies:
-  - name: lens
-    version: 0.1.0-dev.0
-    repository: oci://ghcr.io/berriai/charts

helm/litellm/README.md

@@ -1,10 +1,8 @@
 # LiteLLM componentized chart
 
-This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
+This chart deploys separate gateway, backend and UI services
 
-Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
-
-Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
 
 For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
 

helm/litellm/templates/_helpers.tpl

@@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
-{{- define "litellm.lensWorker.image" -}}
-{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
-{{- end -}}
-
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@@ -635,93 +631,27 @@ shutdown drain window.
 {{- end -}}
 
 {{- define "litellm.lensConnectionEnv" -}}
-{{- $mode := include "litellm.lens.mode" . -}}
-{{- if ne $mode "disabled" }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
-{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
+{{- with .Values.lensWorker }}
+{{- if or .enabled (and .mode (ne .mode "disabled")) }}
+{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
 {{- end }}
-{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
-{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
-{{- end -}}
+{{- end }}
+{{- with .Values.lens }}
+{{- if .enabled }}
 - name: LITELLM_LENS_URL
-  value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
+  value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
 - name: LITELLM_LENS_PUBLIC_URL
-  value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
+  value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
 - name: LENS_GATEWAY_SECRET
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
-      key: {{ .Values.lensWorker.gateway.secretKey | quote }}
+      name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
+      key: {{ .gatewaySecret.key | quote }}
 - name: LITELLM_LENS_SERVICE_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
-      key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
+      name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
+      key: {{ .serviceTokenSecret.key | quote }}
+{{- end }}
 {{- end }}
 {{- end -}}
-
-{{- define "litellm.lensWorker.labels" -}}
-{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
-{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
-{{- toYaml $labels -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
-{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.gatewaySecretName" -}}
-{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.bundledClickhouse" -}}
-{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.publicUrl" -}}
-{{- if .Values.lensWorker.publicUrl -}}
-{{- .Values.lensWorker.publicUrl -}}
-{{- else if eq (include "litellm.lens.mode" .) "external" -}}
-{{- fail "lensWorker.publicUrl is required for external Lens" -}}
-{{- else if .Values.lensWorker.ingress.enabled -}}
-{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
-{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
-{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
-{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
-{{- else -}}
-{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "litellm.lensWorker.clickhouseName" -}}
-{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
-{{- end -}}
-
-{{- define "litellm.lens.mode" -}}
-{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
-{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
-{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
-{{- end -}}
-{{- $mode -}}
-{{- end -}}
-
-{{- define "litellm.lens.render" -}}
-{{- $root := .root -}}
-{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
-{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
-{{- $_ := set $values "component" "lens-worker" -}}
-{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
-{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
-{{- $_ := set $values.gateway "enabled" true -}}
-{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
-{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
-{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
-{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
-{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
-{{- $_ := set $values "ingestionUrl" $ingestion -}}
-{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
-{{- end -}}
-{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
-{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -166,14 +166,14 @@ spec:
     - host: {{ $componentHosts.gateway | quote }}
       http:
         paths:
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           - path: /
             pathType: Prefix
@@ -249,15 +249,15 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
-          {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
+          {{- if and .Values.lens.enabled .Values.lens.service.name }}
           {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
           - path: /lens-ingest
             pathType: Prefix
             backend:
               service:
-                name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
+                name: {{ .Values.lens.service.name }}
                 port:
-                  number: {{ .Values.lensWorker.service.port }}
+                  number: {{ .Values.lens.service.port }}
           {{- end }}
           {{- /*
             --- Operator-supplied extra paths (ingress.extraPaths) ---

helm/litellm/templates/lens/clickhouse.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
-{{- end }}

helm/litellm/templates/lens/deployment.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
-{{- end }}

helm/litellm/templates/lens/ingress.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
-{{- end }}

helm/litellm/templates/lens/secrets.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
-{{- end }}

helm/litellm/templates/lens/service.yaml

@@ -1,3 +0,0 @@
-{{- if eq (include "litellm.lens.mode" .) "bundled" }}
-{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
-{{- end }}

helm/litellm/tests/component_hosts_tests.yaml

@@ -15,7 +15,6 @@ tests:
       ingress.componentHosts.ui: ui.example.com
       ingress.componentHosts.backend: backend.example.com
       ingress.componentHosts.gateway: gateway.example.com
-      lensWorker.mode: disabled
       ingress.tls:
         - hosts:
             - ui.example.com
@@ -83,12 +82,12 @@ tests:
       ui.backendUrl: https://backend.example.com
       ui.publicUrl: https://ui.example.com
       backend.gatewayUrl: https://gateway.example.com
-      lensWorker.mode: external
-      lensWorker.externalServiceName: existing-lens
-      lensWorker.externalUrl: https://lens.example.com
-      lensWorker.publicUrl: https://lens.example.com
-      lensWorker.serviceTokenSecret.name: existing-lens-signing
-      lensWorker.gateway.secretName: existing-lens-signing
+      lens.enabled: true
+      lens.service.name: existing-lens
+      lens.url: http://existing-lens:4318
+      lens.publicUrl: https://gateway.example.com/lens-ingest
+      lens.serviceTokenSecret.name: existing-lens-signing
+      lens.gatewaySecret.name: existing-lens-signing
     asserts:
       - equal:
           path: spec.rules[2].http.paths[0].path

helm/litellm/tests/lens_connection_tests.yaml

@@ -0,0 +1,166 @@
+suite: lens connection
+templates:
+  - gateway/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: connects the gateway to an external Lens
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: connects the backend to an external Lens
+    template: backend/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_URL
+            value: http://lens.lens.svc:4318
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_PUBLIC_URL
+            value: https://lens.example.com/lens-ingest
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_GATEWAY_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: lens-gateway
+                key: gateway-secret
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_LENS_SERVICE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: lens-service
+                key: service-token
+
+  - it: requires lens.url when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.url is required when lens.enabled is true
+
+  - it: requires lens.publicUrl when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.publicUrl is required when lens.enabled is true
+
+  - it: requires lens.gatewaySecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
+
+  - it: requires lens.serviceTokenSecret.name when Lens is enabled
+    template: gateway/deployment.yaml
+    set:
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
+
+  - it: routes /lens-ingest to the named Lens service on the single ingress host
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+      lens.service.name: existing-lens
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: existing-lens
+                port:
+                  number: 4318
+
+  - it: leaves /lens-ingest off the ingress without a Lens service name
+    template: ingress.yaml
+    set:
+      ingress.enabled: true
+      ingress.host: litellm.example.com
+      lens.enabled: true
+      lens.url: http://lens.lens.svc:4318
+      lens.publicUrl: https://lens.example.com/lens-ingest
+      lens.gatewaySecret.name: lens-gateway
+      lens.serviceTokenSecret.name: lens-service
+    asserts:
+      - exists:
+          path: spec.rules[0].http.paths
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /lens-ingest
+          any: true

helm/litellm/tests/lens_endpoint_defaults_tests.yaml

@@ -1,86 +0,0 @@
-suite: Lens endpoint defaults
-templates:
-  - gateway/deployment.yaml
-  - gateway/configmap.yaml
-set:
-  lensWorker.enabled: true
-  ingress.enabled: true
-  ingress.host: gateway.example
-  ingress.tls:
-    - hosts: [gateway.example]
-      secretName: tls
-tests:
-  - it: derives the trace endpoint from the deployment hostname and TLS
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://gateway.example/lens-ingest
-  - it: preserves an explicitly configured public address
-    set:
-      lensWorker.publicUrl: https://custom.example/prefix
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://custom.example/prefix
-  - it: uses the dedicated Lens ingress when configured
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls:
-        - hosts: [traces.example]
-          secretName: traces-tls
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTPS for a dedicated ALB ingress with certificate annotations
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.className: alb
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations:
-        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
-        alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
-        alb.ingress.kubernetes.io/ssl-redirect: "443"
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: https://traces.example
-  - it: uses HTTP for a dedicated ingress without its own TLS
-    set:
-      lensWorker.ingress.enabled: true
-      lensWorker.ingress.host: traces.example
-      lensWorker.ingress.tls: []
-      lensWorker.ingress.annotations: {}
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://traces.example
-  - it: uses HTTP when ingress has no TLS
-    set:
-      ingress.tls: []
-    template: gateway/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_LENS_PUBLIC_URL
-            value: http://gateway.example/lens-ingest
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_modes_tests.yaml

@@ -1,208 +0,0 @@
----
-suite: Lens deployment modes
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-- lens/deployment.yaml
-- lens/service.yaml
-- lens/clickhouse.yaml
-- lens/secrets.yaml
-- ingress.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: retains the existing deployment before transferring its release ownership
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.retainResources: true
-  asserts:
-  - equal:
-      path: metadata.annotations["helm.sh/resource-policy"]
-      value: keep
-- it: keeps the ingestion endpoint on an independently deployed Lens service
-  template: ingress.yaml
-  set:
-    ingress.enabled: true
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: existing-lens
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: connects an external deployment without installing Lens resources
-  templates:
-  - lens/deployment.yaml
-  - lens/service.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: sends delegated requests to the explicitly selected external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens.other-namespace:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: external-lens-signing
-            key: gateway-secret
-- it: disables Lens explicitly even with a retained legacy enable flag
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-  asserts:
-  - hasDocuments:
-      count: 0
-- it: bundles Lens through explicit mode selection
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-  asserts:
-  - hasDocuments:
-      count: 1
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_MODE
-        value: standalone
-- it: uses separate credentials for external identity and service access
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.serviceTokenSecret: {name: worker-service, key: token}
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_SERVICE_TOKEN
-        valueFrom:
-          secretKeyRef: {name: worker-service, key: token}
-- it: supplies the same separate signing secret to the bundled service
-  template: lens/deployment.yaml
-  set:
-    lensWorker.mode: bundled
-    lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef: {name: delegated-identity, key: signature}
-- it: routes explicit bundled mode with the enable flag off
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: bundled
-    lensWorker.enabled: false
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: gateway-lens-worker
-            port: {number: 4318}
-- it: routes external mode to its selected service
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: "existing-lens"
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: existing-lens
-            port: {number: 4318}
-- it: omits ingestion when an external service has no ingress backend
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: external
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true
-- it: omits ingestion for disabled mode with the enable flag on
-  template: ingress.yaml
-  set:
-    fullnameOverride: gateway
-    ingress.enabled: true
-    ingress.host: gateway.example
-    lensWorker.mode: disabled
-    lensWorker.enabled: true
-    lensWorker.externalUrl: http://existing-lens:4318
-    lensWorker.externalServiceName: ""
-    lensWorker.serviceTokenSecret.name: worker-service
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - notContains:
-      path: spec.rules[0].http.paths
-      content: {path: /lens-ingest}
-      any: true

helm/litellm/tests/lens_modes_validation_tests.yaml

@@ -1,54 +0,0 @@
----
-suite: Lens deployment mode validation
-templates:
-- gateway/configmap.yaml
-- backend/deployment.yaml
-values:
-- "./values/required.yaml"
-set:
-  lensWorker.publicUrl: https://traces.example
-tests:
-- it: requires a separate identity signing reference for external Lens
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: worker-service
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.gateway.secretName is required for external Lens
-- it: requires an external service address
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.externalUrl is required for external Lens
-- it: requires the shared server credential for an external service
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
-- it: requires an agent reachable URL for external ingestion
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: external
-    lensWorker.externalUrl: http://lens.other-namespace:4318
-    lensWorker.serviceTokenSecret.name: external-lens-signing
-    lensWorker.gateway.secretName: external-lens-signing
-    lensWorker.publicUrl: ''
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required for external Lens
-- it: rejects unsupported deployment modes
-  template: backend/deployment.yaml
-  set:
-    lensWorker.mode: typo
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.mode must be bundled, external or disabled

helm/litellm/tests/lens_saved_secrets_tests.yaml

@@ -1,61 +0,0 @@
-suite: Lens preserves credentials across Helm upgrades
-templates:
-  - lens/secrets.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-release:
-  namespace: lens
-  name: lens-test
-  upgrade: true
-kubernetesProvider:
-  scheme:
-    v1/Secret:
-      gvr:
-        version: v1
-        resource: secrets
-      namespaced: true
-  objects:
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-gateway
-        namespace: lens
-      data:
-        gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-service
-        namespace: lens
-      data:
-        service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
-    - apiVersion: v1
-      kind: Secret
-      metadata:
-        name: lens-test-lens-clickhouse
-        namespace: lens
-      data:
-        password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-tests:
-  - it: preserves the separate signing credential across upgrades
-    documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
-    asserts:
-      - equal:
-          path: data.gateway-secret
-          value: c2F2ZWQtc2lnbmluZy1rZXk=
-      - notExists: {path: data.service-token}
-  - it: reuses the service credential instead of breaking running services
-    documentIndex: 0
-    asserts:
-      - equal:
-          path: data.service-token
-          value: c2F2ZWQtc2VydmljZS10b2tlbg==
-  - it: reuses the database password instead of locking out stored traces
-    documentIndex: 1
-    asserts:
-      - equal:
-          path: data.password
-          value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_service_tests.yaml

@@ -1,196 +0,0 @@
-suite: Lens service isolation and ingestion routing
-templates:
-- gateway/configmap.yaml
-- gateway/deployment.yaml
-- backend/deployment.yaml
-- ingress.yaml
-- lens/ingress.yaml
-- lens/service.yaml
-- lens/deployment.yaml
-tests:
-- it: connects gateway/deployment.yaml to the shared Lens service
-  template: gateway/deployment.yaml
-  set: &id001
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: connects backend/deployment.yaml to the shared Lens service
-  template: backend/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-        value: http://lens-test-lens-worker:4318
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_PUBLIC_URL
-        value: https://gateway.example/lens-ingest
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LENS_GATEWAY_SECRET
-        valueFrom:
-          secretKeyRef:
-            name: lens-test-lens-gateway
-            key: gateway-secret
-- it: routes uploads directly to Lens instead of the gateway
-  template: ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    ingress.enabled: true
-    ingress.host: gateway.example
-  asserts:
-  - contains:
-      path: spec.rules[0].http.paths
-      content:
-        path: /lens-ingest
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              number: 4318
-- it: keeps internal routes out of a dedicated ingestion hostname
-  template: lens/ingress.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://gateway.example/lens-ingest
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.ingress.enabled: true
-    lensWorker.ingress.host: traces.example
-  asserts:
-  - equal:
-      path: spec.rules[0].http.paths
-      value:
-      - path: /v1/
-        pathType: Prefix
-        backend:
-          service:
-            name: lens-test-lens-worker
-            port:
-              name: otlp
-- it: maps the Lens service to the ingestion listener
-  template: lens/service.yaml
-  set: *id001
-  asserts:
-  - equal:
-      path: spec.selector
-      value:
-        app.kubernetes.io/instance: RELEASE-NAME
-        app.kubernetes.io/component: lens-worker
-  - equal:
-      path: spec.ports
-      value:
-      - name: otlp
-        port: 4318
-        targetPort: otlp
-- it: gives only Lens the ClickHouse secret
-  template: lens/deployment.yaml
-  set: *id001
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_URL
-        valueFrom:
-          secretKeyRef:
-            name: lens-storage
-            key: url
-- it: requires an agent reachable ingestion URL
-  template: gateway/deployment.yaml
-  set:
-    fullnameOverride: lens-test
-    lensWorker.enabled: true
-    lensWorker.publicUrl: ''
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - failedTemplate:
-      errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
-- it: omits Lens connection settings when disabled in gateway/deployment.yaml
-  template: gateway/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: omits Lens connection settings when disabled in backend/deployment.yaml
-  template: backend/deployment.yaml
-  asserts:
-  - notContains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: LITELLM_LENS_URL
-      any: true
-- it: preserves an existing ClickHouse database and retention
-  template: lens/deployment.yaml
-  set:
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-    lensWorker.clickhouseDatabase: existing_traces
-    lensWorker.retentionDays: 45
-  asserts:
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: CLICKHOUSE_DATABASE
-        value: existing_traces
-  - contains:
-      path: spec.template.spec.containers[0].env
-      content:
-        name: AGENT_TRACING_RETENTION_DAYS
-        value: '45'
-- it: keeps Lens pods outside the gateway autoscaling selector
-  template: lens/deployment.yaml
-  set: &id002
-    nameOverride: inference
-    lensWorker.enabled: true
-    lensWorker.publicUrl: https://traces.example
-    lensWorker.serviceTokenSecret.name: lens-service
-    lensWorker.clickhouseSecret.name: lens-storage
-  asserts:
-  - equal:
-      path: spec.template.metadata.labels["app.kubernetes.io/name"]
-      value: inference-lens-worker
-- it: preserves the existing gateway deployment selector
-  template: gateway/deployment.yaml
-  set: *id002
-  asserts:
-  - equal:
-      path: spec.selector.matchLabels["app.kubernetes.io/name"]
-      value: inference
-values:
-- ./values/required.yaml

helm/litellm/tests/lens_setup_tests.yaml

@@ -1,164 +0,0 @@
-suite: Lens managed setup
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-release:
-  namespace: lens
-  name: lens-test
-templates:
-  - lens/deployment.yaml
-  - lens/clickhouse.yaml
-  - lens/secrets.yaml
-tests:
-  - it: generates private login, gateway and storage credentials
-    template: lens/secrets.yaml
-    asserts:
-      - hasDocuments:
-          count: 4
-      - matchRegex:
-          path: data.service-token
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 0
-      - matchRegex:
-          path: data.password
-          pattern: '^[A-Za-z0-9+/]{86}==$'
-        documentIndex: 1
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
-  - it: connects Lens to its private bundled storage
-    template: lens/deployment.yaml
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: lens-test-lens-clickhouse
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: stores traces on a persistent volume with the chosen storage class
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      lensWorker.clickhouse.storage: 40Gi
-      lensWorker.clickhouse.storageClassName: fast
-    asserts:
-      - equal:
-          path: spec.volumeClaimTemplates[0].spec
-          value:
-            accessModes: [ReadWriteOnce]
-            storageClassName: fast
-            resources:
-              requests:
-                storage: 40Gi
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: lens-test-lens-clickhouse
-                key: password
-  - it: keeps an existing external database instead of creating a new one
-    template: lens/clickhouse.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: leaves supplied secrets under their existing manager
-    template: lens/secrets.yaml
-    set:
-      lensWorker.clickhouseSecret.name: external-clickhouse
-      lensWorker.serviceTokenSecret.name: external-service
-      lensWorker.adminTokenSecret.name: external-admin
-      lensWorker.gateway.secretName: external-signing
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: creates no credentials or database when Lens is disabled
-    templates:
-      - lens/secrets.yaml
-      - lens/clickhouse.yaml
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments:
-          count: 0
-  - it: requires an external database when bundled storage is explicitly disabled
-    template: lens/deployment.yaml
-    set:
-      lensWorker.clickhouse.enabled: false
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
-  - it: keeps storage names valid and uses the same name for the connection
-    set:
-      fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - matchRegex:
-          path: metadata.name
-          pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 0
-      - equal:
-          path: spec.serviceName
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/clickhouse.yaml
-        documentIndex: 1
-      - equal:
-          path: metadata.name
-          value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/secrets.yaml
-        documentIndex: 1
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_HOST
-            value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-        template: lens/deployment.yaml
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: CLICKHOUSE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
-                key: password
-        template: lens/deployment.yaml
-  - it: allows cold storage startup and inherits registry credentials
-    template: lens/clickhouse.yaml
-    documentIndex: 1
-    set:
-      imagePullSecrets: [{name: registry-auth}]
-    asserts:
-      - equal:
-          path: spec.template.spec.imagePullSecrets
-          value: [{name: registry-auth}]
-      - equal:
-          path: spec.template.spec.containers[0].startupProbe
-          value:
-            httpGet: {path: /ping, port: http}
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
-
-values:
-  - ./values/required.yaml

helm/litellm/tests/lens_worker_tests.yaml

@@ -1,79 +0,0 @@
-suite: Independent Lens release and credentials
-templates:
-  - lens/deployment.yaml
-values:
-  - ./values/required.yaml
-set:
-  fullnameOverride: lens-test
-  lensWorker.enabled: true
-  lensWorker.publicUrl: https://traces.example
-  lensWorker.serviceTokenSecret.name: lens-service
-  lensWorker.clickhouseSecret.name: lens-storage
-tests:
-  - it: selects a Lens release independently of the gateway
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-      lensWorker.image.tag: 2.3.4
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:2.3.4
-  - it: retains the pinned Lens chart default when only the gateway changes
-    chart:
-      appVersion: 9.8.7
-    set:
-      backend.image.tag: gateway-9.8.7
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens:0.1.0-dev.0
-  - it: selects development publications explicitly
-    set:
-      lensWorker.image.repository: ghcr.io/berriai/lens-dev
-      lensWorker.image.tag: sha-0123456789abcdef
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
-  - it: preserves a verified registry mirror and digest
-    set:
-      lensWorker.image.repository: registry.example/lens
-      lensWorker.image.tag: replaced-label
-      lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].image
-          value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-  - it: rejects a malformed digest
-    set:
-      lensWorker.image.digest: sha256:invalid
-    asserts:
-      - failedTemplate:
-          errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
-  - it: keeps the legacy enablement switch working
-    set:
-      lensWorker.enabled: false
-    asserts:
-      - hasDocuments: {count: 0}
-  - it: keeps isolated local analysis with delegated gateway access
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content: {name: LENS_MODE, value: standalone}
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LENS_GATEWAY_SECRET
-            valueFrom:
-              secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
-      - equal:
-          path: spec.template.spec.automountServiceAccountToken
-          value: false
-      - equal:
-          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
-          value: true
-      - equal:
-          path: spec.template.spec.volumes[0].emptyDir
-          value: {medium: Memory, sizeLimit: 1Gi}

helm/litellm/values.yaml

@@ -654,70 +654,15 @@ ui:
   topologySpreadConstraints: []
 
 lens:
-  library: true
-
-lensWorker:
-  retainResources: false
-  mode: ""
-  externalUrl: ""
-  externalServiceName: ""
-  gateway:
-    secretName: ""
-    secretKey: gateway-secret
-  standaloneUrl: ""
-  adminTokenSecret:
-    name: ""
-    key: admin-token
-  extraEnv: []
-  extraEnvFrom: []
   enabled: false
-  replicaCount: 1
-  image:
-    repository: ghcr.io/berriai/lens
-    tag: "0.1.0-dev.0"
-    digest: ""
-    pullPolicy: IfNotPresent
-  tokenSecret:
+  url: ""
+  publicUrl: ""
+  service:
     name: ""
-    key: token
+    port: 4318
+  gatewaySecret:
+    name: ""
+    key: gateway-secret
   serviceTokenSecret:
     name: ""
     key: service-token
-  clickhouse:
-    enabled: true
-    image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
-    storage: 20Gi
-    storageClassName: null
-    resources:
-      requests:
-        cpu: 100m
-        memory: 512Mi
-      limits:
-        memory: 2Gi
-  clickhouseDatabase: litellm
-  retentionDays: 14
-  clickhouseSecret:
-    name: ""
-    key: url
-  publicUrl: ""
-  service:
-    port: 4318
-    annotations: {}
-  ingress:
-    enabled: false
-    path: /v1/
-    className: ""
-    host: ""
-    annotations: {}
-    tls: []
-  url: ""
-  tmpSizeLimit: 1Gi
-  resources:
-    requests:
-      cpu: 100m
-      memory: 256Mi
-    limits:
-      memory: 2Gi
-  nodeSelector: {}
-  tolerations: []
-  affinity: {}

litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py

@@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it.
 """
 
 import os
-import re
-from collections.abc import Callable, Iterator
+from collections.abc import Callable
 from dataclasses import dataclass
 from functools import reduce
-from itertools import accumulate
 from pathlib import Path
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal
@@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME
 DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
 REFUSAL: Final = "Existing rows would make a pending schema change fail:"
 
-Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
+Kind = Literal["too_long"]
 
 
 @dataclass(frozen=True, slots=True)
@@ -54,9 +52,6 @@ class Check:
     statement: str
     source: str
     index: str | None = None
-    referenced_table: str | None = None
-    referenced_columns: tuple[str, ...] = ()
-    expression: str | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -100,33 +95,6 @@ class ColumnType:
 
 ColumnTypeLookup = Callable[[str, str], ColumnType | None]
 
-_IDENT: Final = r'"?(\w+)"?'
-_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
-_CREATE_INDEX: Final = re.compile(
-    rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
-    rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
-    re.IGNORECASE | re.DOTALL,
-)
-_ALTER_TABLE: Final = re.compile(
-    rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
-)
-_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
-_ADD_COLUMN: Final = re.compile(
-    rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
-    re.IGNORECASE | re.DOTALL,
-)
-_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
-_ADD_FOREIGN_KEY: Final = re.compile(
-    rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
-    re.IGNORECASE | re.DOTALL,
-)
-_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
-_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
-_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
-_COMMENT: Final = re.compile(r"--[^\n]*")
-_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
-_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
-
 
 def migration_data_check_timeout_seconds() -> float:
     """Seconds one count query may run before the preflight gives up on verifying it."""
@@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float:
         return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
 
 
-def statements(migration_sql: str) -> tuple[str, ...]:
-    """The statements of one migration file, comments stripped, empty ones dropped."""
-    parts: Final = _COMMENT.sub("", migration_sql).split(";")
-    return tuple(part.strip() for part in parts if part.strip())
-
-
-def _split_top_level(text: str) -> tuple[str, ...]:
-    """Split on the commas outside parentheses."""
-    depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
-    cuts: Final = tuple(
-        position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
-    )
-    starts: Final = (0, *(cut + 1 for cut in cuts))
-    stops: Final = (*cuts, len(text))
-    pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
-    return tuple(piece for piece in pieces if piece)
-
-
-def _plain_columns(column_list: str) -> tuple[str, ...] | None:
-    """The column names of an index or constraint column list, None when one entry is an
-    expression the derivation cannot turn into a count."""
-    matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
-    if any(match is None for match in matches):
-        return None
-    return tuple(match.group(1) for match in matches if match is not None)
-
-
-def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
-    unique, name, table, method, column_list = match.groups()
-    columns: Final = _plain_columns(column_list)
-    if columns is None:
-        return
-    label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
-    if method is None or method.lower() == "btree":
-        yield Check("too_long", table, columns, label, source, index=name)
-    if unique:
-        yield Check("duplicates", table, columns, label, source, index=name)
-
-
-def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
-    if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
-        yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
-        return
-    if (add_column := _ADD_COLUMN.match(action)) is not None:
-        column, body = add_column.groups()
-        if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
-            yield Check(
-                "any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
-            )
-        return
-    if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
-        constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
-        own: Final = _plain_columns(columns)
-        theirs: Final = _plain_columns(referenced_columns)
-        if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
-            return
-        yield Check(
-            "orphans",
-            table,
-            own,
-            _constraint_label(table, constraint, "FOREIGN KEY"),
-            source,
-            referenced_table=referenced_table,
-            referenced_columns=theirs,
-        )
-        return
-    if (unique := _ADD_UNIQUE.match(action)) is not None:
-        constraint, columns = unique.groups()
-        unique_columns: Final = _plain_columns(columns)
-        if unique_columns is not None:
-            yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
-        return
-    if (check := _ADD_CHECK.match(action)) is not None:
-        constraint, expression, not_valid = check.groups()
-        if not_valid is None:
-            yield Check(
-                "check_violations",
-                table,
-                (),
-                _constraint_label(table, constraint, "CHECK"),
-                source,
-                expression=expression,
-            )
-
-
-def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
-    named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
-    return f'ALTER TABLE "{table}" ADD {named}{kind}'
-
-
-def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
-    """The count queries one SQL statement needs before it can be applied, empty for a
-    statement that cannot fail on existing rows or that the derivation cannot read."""
-    if (index := _CREATE_INDEX.match(statement_text)) is not None:
-        return tuple(_index_checks(index, statement_text, source))
-    if (alter := _ALTER_TABLE.match(statement_text)) is not None:
-        table, actions = alter.groups()
-        return tuple(
-            check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
-        )  # comprehension-ok: flattening the per-action checks of one statement
-    return ()
-
-
-def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
-    return tuple(
-        check
-        for statement_text in statements(migration_sql)
-        for check in checks_for_statement(statement_text, f"migration {name}")
-    )  # comprehension-ok: flattening the per-statement checks of one migration
-
-
 def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
     """The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
     return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
@@ -424,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...],
 
 
 def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
-    from psycopg import sql
-
-    table: Final = sql.Identifier(schema, check.table)
-    columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
-    match check.kind:
-        case "too_long":
-            return _entry_size_query(check, schema, types, limit_bytes)
-        case "duplicates":
-            not_null: Final = sql.SQL(" AND ").join(
-                sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
-            )
-            return sql.SQL(
-                "SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
-            ).format(t=table, nn=not_null, cols=columns)
-        case "nulls":
-            return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
-                t=table, col=sql.Identifier(check.columns[0])
-            )
-        case "any_row":
-            return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
-        case "orphans":
-            pairs: Final = sql.SQL(" AND ").join(
-                sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
-                for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
-            )
-            present: Final = sql.SQL(" AND ").join(
-                sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
-            )
-            return sql.SQL(
-                "SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
-            ).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
-        case "check_violations":
-            return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
-                t=table,
-                expr=sql.SQL(check.expression or "true"),  # pyright: ignore[reportArgumentType]  # copied from our own migration file
-            )
+    return _entry_size_query(check, schema, types, limit_bytes)
 
 
 def _applies(
@@ -474,15 +296,7 @@ def _applies(
         return False
     if check.index is not None and _index_is_valid(connection, schema, check.index):
         return False
-    if check.kind == "any_row":
-        return types is None
-    if types is None:
-        return False
-    if check.kind == "too_long":
-        return any(column.can_overflow_btree for column in types)
-    if check.kind == "orphans":
-        return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
-    return True
+    return types is not None and any(column.can_overflow_btree for column in types)
 
 
 def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
@@ -515,21 +329,7 @@ def _run_check(
     rows: Final = _int(values[0], 0) if values else 0
     if rows == 0:
         return None
-    match check.kind:
-        case "too_long":
-            return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
-        case "duplicates":
-            return BlockedChange(check, rows, "duplicated value groups")
-        case "nulls":
-            return BlockedChange(check, rows, "NULL")
-        case "any_row":
-            return BlockedChange(
-                check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
-            )
-        case "orphans":
-            return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
-        case "check_violations":
-            return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
+    return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
 
 
 def find_blockers(
@@ -571,19 +371,7 @@ def find_blockers(
 
 def _remedy(check: Check) -> str:
     joined: Final = ", ".join(check.columns)
-    match check.kind:
-        case "too_long":
-            return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
-        case "duplicates":
-            return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
-        case "nulls":
-            return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
-        case "any_row":
-            return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
-        case "orphans":
-            return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
-        case "check_violations":
-            return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
+    return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
 
 
 def _describe(finding: Finding) -> str:
@@ -607,30 +395,4 @@ def _describe(finding: Finding) -> str:
 def report(findings: tuple[Finding, ...]) -> str:
     """The operator-facing refusal: one block per finding, then what to do next."""
     lines: Final = "\n".join(_describe(finding) for finding in findings)
-    return (
-        f"{REFUSAL}\n{lines}\n"
-        "No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
-    )
-
-
-def pending_change_checks(
-    connection: "psycopg.Connection[tuple[object, ...]]",
-    schema: str,
-    migrations_dir: Path | None,
-    indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
-) -> tuple[Check, ...]:
-    """Every check the pending migrations and the request-log indexes need. `migrations_dir`
-    is None when the schema is pushed rather than migrated, so only the indexes are checked."""
-    migration_checks: Final = (
-        tuple(
-            check
-            for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
-            for check in checks_for_migration(name, migration_sql)
-        )  # comprehension-ok: flattening the per-migration checks
-        if migrations_dir is not None
-        else ()
-    )
-    index_checks: Final = tuple(
-        check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
-    )  # comprehension-ok: flattening the per-index checks
-    return migration_checks + index_checks
+    return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up."

litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py

@@ -0,0 +1,184 @@
+"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each
+inside its own transaction that is always rolled back, so Postgres itself says whether this
+database's rows and schema take them. Each migration holds the locks the real upgrade will,
+and releases them before the next one runs."""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import reduce
+from pathlib import Path
+from typing import TYPE_CHECKING, Final
+
+from litellm_proxy_extras.migration_data_checks import pending_migrations
+
+if TYPE_CHECKING:
+    import psycopg
+
+_CONCURRENTLY: Final = re.compile(
+    r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE
+)
+UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"})
+
+Migration = tuple[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class Applied:
+    name: str
+    seconds: float
+    replayed: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class Rejected:
+    name: str
+    error: str
+    seconds: float
+
+
+@dataclass(frozen=True, slots=True)
+class Baseline:
+    """A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying
+    history, so the dry run asks the same question the boot does. `refusal` is None when the boot
+    will baseline, otherwise its reason (a custom migration folder, or the schema diff)."""
+
+    refusal: str | None
+
+    @property
+    def matches(self) -> bool:
+        return self.refusal is None
+
+
+@dataclass(frozen=True, slots=True)
+class DryRun:
+    applied: tuple[Applied, ...]
+    rejected: Rejected | None
+    baseline: Baseline | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.rejected is None and (self.baseline is None or self.baseline.matches)
+
+
+def transactional(migration_sql: str) -> str:
+    """The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones."""
+    return _CONCURRENTLY.sub(r"\1", migration_sql)
+
+
+def _attempt(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    chain: tuple[Migration, ...],
+    clock: Callable[[], float],
+) -> tuple[float, psycopg.Error | None]:
+    import psycopg
+    from psycopg import sql
+
+    started: Final = clock()
+    try:
+        with connection.transaction(force_rollback=True):
+            connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema)))
+            for _, migration_sql in chain:
+                connection.execute(transactional(migration_sql))
+    except psycopg.Error as exc:
+        return clock() - started, exc
+    return clock() - started, None
+
+
+def _run(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    pending: tuple[Migration, ...],
+    position: int,
+    done: DryRun,
+    clock: Callable[[], float],
+) -> DryRun:
+    """One migration in its own rolled-back transaction. A migration that fails because something
+    an earlier pending migration creates does not exist yet is rerun with those replayed in front."""
+    if done.rejected is not None:
+        return done
+    name: Final = pending[position][0]
+    seconds, error = _attempt(connection, schema, (pending[position],), clock)
+    if error is None:
+        return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline)
+    if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0:
+        return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline)
+    earlier: Final = pending[:position]
+    chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock)
+    if chained_error is None:
+        replayed: Final = tuple(earlier_name for earlier_name, _ in earlier)
+        return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline)
+    return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline)
+
+
+def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
+    from psycopg import sql
+
+    row: Final = connection.execute(
+        "SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),)
+    ).fetchone()
+    return row is not None and row[0] is not None
+
+
+def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
+    row: Final = connection.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1",
+        (schema,),
+    ).fetchone()
+    return row is not None
+
+
+def dry_run(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    migrations_dir: Path,
+    baseline_check: Callable[[], str | None] | None = None,
+    clock: Callable[[], float] = time.monotonic,
+) -> DryRun:
+    """Apply every pending migration in order, each in its own transaction that is rolled back
+    before the next one starts. The first migration Postgres rejects ends the run with its error.
+    `baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for
+    the replay on a database that has tables but no migration history, which the boot baselines."""
+    if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema):
+        return DryRun((), None, Baseline(baseline_check()))
+    pending: Final = pending_migrations(connection, schema, migrations_dir)
+    return reduce(
+        lambda done, position: _run(connection, schema, pending, position, done, clock),
+        range(len(pending)),
+        DryRun((), None),
+    )
+
+
+def _timing(applied: Applied) -> str:
+    replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else ""
+    return f"\n  {applied.name}: {applied.seconds:.1f} s{replay}"
+
+
+def report(result: DryRun) -> str:
+    """What the operator reads: the per-migration time the real upgrade will hold its locks,
+    or the migration Postgres rejected with Postgres's own words. Nothing was applied either way."""
+    if result.baseline is not None:
+        if result.baseline.matches:
+            return (
+                "No migration history and the schema matches this build, "
+                "the proxy baselines it at boot, nothing to dry-run."
+            )
+        return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}"
+    timings: Final = "".join(_timing(applied) for applied in result.applied)
+    if result.rejected is None:
+        count: Final = len(result.applied)
+        if count == 0:
+            return "No pending migrations, this database is on this version's schema."
+        return (
+            f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next "
+            f"so its locks are released, nothing changed:{timings}"
+        )
+    return (
+        f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, "
+        f"nothing was applied.\n{result.rejected.error}"
+        + (f"\nApplied before it in the dry run:{timings}" if timings else "")
+    )

litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py

@@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
     )
 
 
+def has_packaged_history(migrations_dir: Path) -> bool:
+    """Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships,
+    the only history the boot baselines an existing database onto."""
+    packaged_dir: Final = Path(__file__).parent
+    migrations: Final = migration_files(migrations_dir)
+    return (
+        bool(migrations)
+        and migrations == migration_files(packaged_dir)
+        and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes()
+    )
+
+
 def baseline_current_schema(
     coordinator: MigrationCoordinator,
     schema: str,
@@ -95,14 +107,9 @@ def baseline_current_schema(
 ) -> None:
     from psycopg import sql
 
-    packaged_dir: Final = Path(__file__).parent
-    migrations: Final = migration_files(migrations_dir)
-    if (
-        not migrations
-        or migrations != migration_files(packaged_dir)
-        or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
-    ):
+    if not has_packaged_history(migrations_dir):
         raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
+    migrations: Final = migration_files(migrations_dir)
 
     coordinator.acquire_prisma_lock()
     existing: Final = coordinator.connection.execute(

litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql

@@ -0,0 +1,21 @@
+SET LOCAL lock_timeout = '10s';
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
+    "subscription_account_id" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "account_id" TEXT NOT NULL,
+    "label" TEXT,
+    "monthly_fee" DOUBLE PRECISION NOT NULL,
+    "currency" TEXT NOT NULL,
+    "billing_period_start" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT NOT NULL,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -11,18 +11,13 @@ from collections.abc import Callable
 from contextlib import AbstractContextManager, nullcontext
 from dataclasses import dataclass, replace
 from pathlib import Path
-from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
+from typing import TYPE_CHECKING, Final, Protocol
 
 from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras._redaction import _redact_credentials
-from litellm_proxy_extras.migration_data_checks import (
-    Finding,
-    find_blockers,
-    migration_data_check_timeout_seconds,
-    pending_change_checks,
-    report,
-)
+from litellm_proxy_extras.migration_dry_run import dry_run
+from litellm_proxy_extras.migration_dry_run import report as dry_run_report
 from litellm_proxy_extras.migration_lock import held_migration_lock
 from litellm_proxy_extras.prisma_toolchain import (
     PRISMA_COMMAND_TIMEOUT_ENV_VAR,
@@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import (
     apply_replica_identity_full,
 )
 from litellm_proxy_extras.request_log_indexes import (
-    REQUEST_LOG_INDEXES,
     ensure_request_log_indexes,
     filter_request_log_index_diff,
 )
@@ -52,7 +46,7 @@ class LensCheckConnect(Protocol):
     ) -> "psycopg.Connection[tuple[object, ...]]": ...
 
 
-def str_to_bool(value: Optional[str]) -> bool:
+def str_to_bool(value: str | None) -> bool:
     if value is None:
         return False
     return value.lower() in ("true", "1", "t", "y", "yes")
@@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str:
     ).strip()
 
 
-def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+def _without_spend_logs_pk_clauses(statement: str) -> str | None:
     prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
     if not prefix_match:
         return statement
@@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int:
     return max(_migration_timestamp(n) for n in names)
 
 
-def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
+def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]:
     if isinstance(command, tuple):
         return tuple(_redact_credentials(str(argument)) for argument in command)
     if isinstance(command, list):
@@ -404,7 +398,7 @@ class ProxyExtrasDBManager:
         return tuple(row) if row is not None else ()
 
     @staticmethod
-    def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
+    def _failed_migration_logs(migration_name: str, started_at: str) -> str | None:
         row: Final = ProxyExtrasDBManager._read_migration_ledger(
             "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
             "AND finished_at IS NULL AND rolled_back_at IS NULL",
@@ -721,10 +715,10 @@ class ProxyExtrasDBManager:
         return row is not None
 
     @staticmethod
-    def _prisma_schema_param(url: str) -> Optional[str]:
+    def _prisma_schema_param(url: str) -> str | None:
         """The `schema` query param Prisma uses to pick its target schema,
         or None when the URL does not set one."""
-        from urllib.parse import urlparse, parse_qsl
+        from urllib.parse import parse_qsl, urlparse
 
         return next(
             (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
@@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager:
             bool: True if setup was successful, False otherwise
         """
         ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
-        ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
         migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
         if not migrated:
             return False
@@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager:
         return True
 
     @staticmethod
-    def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
-        """What in the existing rows would make a pending migration or request-log index
-        fail, read with count queries only: no write, no table lock. Empty when nothing
-        blocks, when no database is configured, or when it cannot be reached, so the
-        migration step that follows reports the connection problem the way it does today."""
+    def _baseline_refusal(prisma_dir: Path) -> str | None:
+        """Why `baseline_current_schema` would refuse this database at boot, None when it would baseline:
+        the migrations folder must be the packaged one, then `prisma migrate diff` must find the live
+        database and this build's schema.prisma identical."""
+        from litellm_proxy_extras.migration_lock import migration_environment
+        from litellm_proxy_extras.migration_recovery import has_packaged_history
+
+        if not has_packaged_history(prisma_dir):
+            return (
+                f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the "
+                "migrations this build ships, and the proxy only baselines onto the packaged history."
+            )
+        try:
+            prisma_toolchain.run_prisma(
+                (
+                    _get_prisma_command(),
+                    "migrate",
+                    "diff",
+                    "--from-schema-datasource",
+                    str(prisma_dir / "schema.prisma"),
+                    "--to-schema-datamodel",
+                    str(prisma_dir / "schema.prisma"),
+                    "--exit-code",
+                ),
+                timeout=prisma_toolchain.prisma_command_timeout(),
+                env=migration_environment(_get_prisma_env()),
+            )
+        except subprocess.CalledProcessError as exc:
+            diff: Final = str(exc.stdout or exc.stderr or exc).strip()
+            return f"the schema differs from this build. prisma migrate diff:\n{diff}"
+        except subprocess.TimeoutExpired as exc:
+            return f"prisma migrate diff did not finish within {exc.timeout} s"
+        return None
+
+    @staticmethod
+    def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
+        """`--check_migrations_against_existing_data`: the pending migrations applied for real on
+        DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports
+        what the upgrade would hit. (True, the per-migration timings) when they all apply,
+        (False, Postgres's own error) when one does not."""
         import psycopg
 
+        if not use_migrate:
+            return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.")
         database_url: Final = os.environ.get("DATABASE_URL")
         if not database_url:
-            return ()
+            return (False, "DATABASE_URL is not set.")
         direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
             os.environ.get("DIRECT_URL") or database_url
         )
         schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
-        migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
-        indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
+        prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir())
         try:
             with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
-                checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
-                return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
+                result: Final = dry_run(
+                    connection,
+                    schema,
+                    prisma_dir / "migrations",
+                    baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir),
+                )
         except psycopg.OperationalError as exc:
-            logger.warning(
-                "Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
-            )
-            return ()
-
-    @staticmethod
-    def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
-        """The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
-        findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
-        return report(findings) if findings else None
-
-    @staticmethod
-    def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
-        """Refuse to run migrations that the existing rows would make fail, before any is
-        applied. The RuntimeError carries the report and reaches the operator through the
-        proxy's "Database migration cannot proceed" exit."""
-        refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
-        if refusal is not None:
-            raise RuntimeError(refusal)
+            return (False, f"Could not reach the database: {_redact_credentials(str(exc))}")
+        return (result.ok, dry_run_report(result))
 
     @staticmethod
     def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
@@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager:
                 time.sleep(random.randrange(5, 15))
             finally:
                 os.chdir(original_dir)
-                pass
         return False

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.111"
+version = "0.4.112"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.111"
+version = "0.4.112"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -4345,6 +4345,7 @@ dependencies = [
  "litellm-inference-transcription",
  "litellm-llms",
  "litellm-llms-types",
+ "litellm-model-catalog",
  "litellm-router-types",
  "litellm-secrets",
  "litellm-secrets-aws",

litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs

@@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[
         |mapping| {
             mapping.error_str.contains("invalid_request_error")
                 && !mapping.error_str.contains("Incorrect API key provided")
+                && !matches!(mapping.status, Some(401 | 404))
         },
         PublicError::BadRequest,
     ),
@@ -132,6 +133,23 @@ mod tests {
         assert_eq!(classified(Some(400), text), Some(expected));
     }
 
+    #[rstest::rstest]
+    #[case::unauthorized(Some(401), None)]
+    #[case::not_found(Some(404), None)]
+    #[case::forbidden(Some(403), Some(PublicError::BadRequest))]
+    #[case::rate_limited(Some(429), Some(PublicError::BadRequest))]
+    #[case::server_error(Some(500), Some(PublicError::BadRequest))]
+    #[case::no_status(None, Some(PublicError::BadRequest))]
+    fn invalid_request_errors_leave_401_and_404_to_the_status_mapping(
+        #[case] status: Option<u16>,
+        #[case] expected: Option<PublicError>,
+    ) {
+        assert_eq!(
+            classified(status, "invalid_request_error bad field"),
+            expected
+        );
+    }
+
     #[rstest::rstest]
     #[case::rate_limit_before_context_window(
         "rate limit and This model's maximum context length is 10",

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -238,7 +238,8 @@ impl AnthropicBeta {
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
-            | Self::ThinkingBindingControls20260801 => &[],
+            | Self::ThinkingBindingControls20260801
+            | Self::ThinkingDisplayUpdates20260818 => &[],
             Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
@@ -275,7 +276,6 @@ impl AnthropicBeta {
                 BetaProvider::Databricks,
             ],
             Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
             Self::ToolSearchTool20251019 => &[
                 BetaProvider::Anthropic,
                 BetaProvider::AzureAi,

litellm-rust/crates/model-catalog/AGENTS.md

@@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
 ## Tests
 
 `BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
+
+`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`

litellm-rust/crates/model-catalog/Cargo.toml

@@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
 
 [dependencies]
 litellm-cost = { workspace = true, features = ["serde"] }
+litellm-error.workspace = true
 
 indexmap = { version = "2.14.0", features = ["serde"] }
 jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
@@ -24,5 +25,4 @@ time.workspace = true
 
 [dev-dependencies]
 jsonschema = { version = "0.55.1", default-features = false }
-litellm-error.workspace = true
 rstest.workspace = true

litellm-rust/crates/model-catalog/src/bin/check-registry.rs

@@ -0,0 +1,26 @@
+use std::{env, fs, path::PathBuf, process::ExitCode};
+
+use litellm_model_catalog::Catalog;
+
+fn main() -> ExitCode {
+    let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
+        eprintln!("usage: check-registry <model_prices_and_context_window.json>");
+        return ExitCode::from(2);
+    };
+    let body = match fs::read(&path) {
+        Ok(body) => body,
+        Err(error) => {
+            eprintln!("{}: {error}", path.display());
+            return ExitCode::FAILURE;
+        }
+    };
+    let problems = Catalog::problems(&body);
+    for problem in &problems {
+        eprintln!("{}", litellm_error::Report::new(problem));
+    }
+    if problems.is_empty() {
+        ExitCode::SUCCESS
+    } else {
+        ExitCode::FAILURE
+    }
+}

litellm-rust/crates/model-catalog/src/catalog.rs

@@ -5,7 +5,7 @@ use indexmap::IndexMap;
 use serde::Deserialize;
 use serde_json::Value;
 
-use crate::error::{Error, Loaded, Rejects, Repr};
+use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
 use crate::fallback::{FallbackGeneralizations, FallbackRule};
 use crate::model::Model;
 use crate::store::{Rows, Store};
@@ -211,6 +211,36 @@ impl Catalog {
         })
     }
 
+    pub fn problems(body: &[u8]) -> Vec<Error> {
+        let Loaded {
+            value: catalog,
+            errors,
+        } = match Self::parse(body) {
+            Ok(loaded) => loaded,
+            Err(fatal) => return vec![fatal],
+        };
+        let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
+            EntryWire::read(entry.fields())
+                .unknown
+                .into_iter()
+                .map(move |(key, _)| {
+                    Error::new(
+                        pointer_child(&pointer_for_key(model), &key),
+                        Repr::UnreadField,
+                    )
+                })
+        });
+        let no_fallback_rules = catalog
+            .fallback_rules()
+            .is_none_or(<[_]>::is_empty)
+            .then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
+        errors
+            .into_iter()
+            .chain(unread)
+            .chain(no_fallback_rules)
+            .collect()
+    }
+
     pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
         let (_, _, errors) = read_row(fields);
         errors
@@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
         )
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
+
+    #[rstest]
+    #[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
+    #[case::not_json("{".to_owned(), vec![("", "json")])]
+    #[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
+    #[case::unread_key(
+        format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
+        vec![("/m/typo_cost", "unread_field")]
+    )]
+    #[case::empty_model_key(
+        format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
+        vec![("//future_field", "unread_field")]
+    )]
+    #[case::unread_key_on_invalid_row(
+        format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
+        vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
+    )]
+    #[case::escaped_unread_key_on_invalid_row(
+        format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
+        vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
+    )]
+    #[case::no_rules(
+        r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
+        vec![("/fallback_generalizations", "no_fallback_rules")]
+    )]
+    #[case::empty_rules(
+        r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
+        vec![("/fallback_generalizations", "no_fallback_rules")]
+    )]
+    fn problems_list_what_a_cost_map_change_must_fix(
+        #[case] body: String,
+        #[case] expected: Vec<(&str, &str)>,
+    ) {
+        let problems: Vec<_> = Catalog::problems(body.as_bytes())
+            .iter()
+            .map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
+            .collect();
+        let expected: Vec<_> = expected
+            .into_iter()
+            .map(|(path, kind)| (path.to_owned(), kind))
+            .collect();
+        assert_eq!(problems, expected);
+    }
+
+    #[rstest]
+    fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
+        let body = br#"{
+            "good":{"litellm_provider":"test","custom_metadata":true},
+            "bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
+        }"#;
+        let loaded = Catalog::parse(body).unwrap();
+        assert_eq!(loaded.errors.len(), 1);
+        assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
+        let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
+        assert_eq!(rejected.len(), 1);
+        assert!(catalog.lookup("bad").is_none());
+        let good = catalog.lookup("good").unwrap();
+        assert!(good.entry.model().is_some());
+        assert_eq!(
+            good.entry.field("custom_metadata"),
+            Some(&Value::Bool(true))
+        );
+        assert!(
+            Catalog::problems(body)
+                .iter()
+                .any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
+        );
+        assert!(
+            Catalog::problems(body)
+                .iter()
+                .any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
+        );
+    }
+}

litellm-rust/crates/model-catalog/src/error.rs

@@ -109,6 +109,12 @@ pub(crate) enum Repr {
         #[source]
         cause: Option<DateCause>,
     },
+    #[error("not a field litellm reads")]
+    #[strum_discriminants(strum(serialize = "unread_field"))]
+    UnreadField,
+    #[error("has no rules")]
+    #[strum_discriminants(strum(serialize = "no_fallback_rules"))]
+    NoFallbackRules,
 }
 
 /// What a problem takes out of the served map
@@ -148,7 +154,7 @@ impl Error {
             | Repr::NoOffPeakHours
             | Repr::EmptyList
             | Repr::InvalidDate { .. } => Rejects::Row,
-            Repr::MissingProvider => Rejects::Nothing,
+            Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
         }
     }
 }
@@ -334,6 +340,8 @@ mod tests {
     #[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
     #[case::empty_list(Repr::EmptyList, Rejects::Row)]
     #[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
+    #[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
+    #[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
     fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
         assert_eq!(Error::from(error).rejects(), expected);
     }

litellm-rust/crates/model-catalog/tests/catalog.rs

@@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() {
 }
 
 #[rstest]
-fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
-    let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
-    let catalog = loaded.value;
-    assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
-    let unrecognized: Vec<_> = catalog.unrecognized().collect();
-    assert!(unrecognized.is_empty(), "{unrecognized:#?}");
-    assert!(
-        catalog
-            .fallback_rules()
-            .is_some_and(|rules| !rules.is_empty())
-    );
+fn the_bundled_registry_has_no_problems() {
+    let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY)
+        .iter()
+        .map(report)
+        .collect();
+    assert!(problems.is_empty(), "{problems:#?}");
     const REFERENCE_MODEL_COUNT: usize = 4303;
+    let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
     assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
 }
 

litellm-rust/crates/python-bridge/AGENTS.md

@@ -106,6 +106,18 @@ GIL handling to `litellm-host-python`.
 
 ## Data Handling
 
+- Reuse large immutable bundled data and parsed catalogs across calls, including in
+  replacement APIs. Share Rust storage and cache repeated Python projections instead
+  of reparsing, serializing or copying the full payload on every access
+- Cache owned Python objects with `PyOnceLock<Py<T>>`, then bind and clone references
+  while attached. Do not use blocking `OnceLock::get_or_init` while attached
+  ([PyO3 initialization](https://pyo3.rs/v0.29.2/faq.html),
+  [reference ownership](https://pyo3.rs/v0.29.2/types.html))
+- Never reuse Python objects across interpreters or interpreter reinitialization.
+  Once-only caches are for immutable data. Reloadable/custom data needs versioned
+  entries or invalidation, and caller-mutable Python containers must not be shared
+- Test unchanged contents and allocation reuse, not just equal values. Keep this
+  coverage when replacing a bundled-data accessor with a catalog API
 - OCR payloads can contain personal data and large base64 images. Do not log
   payloads or provider responses.
 - Avoid copying large payloads more than needed. The current JSON round-trip is

litellm-rust/crates/python-bridge/Cargo.toml

@@ -49,6 +49,7 @@ litellm-secrets-types.workspace = true
 litellm-llms-types.workspace = true
 litellm-router-types.workspace = true
 litellm-host-python.workspace = true
+litellm-model-catalog.workspace = true
 litellm-token-counter = { path = "../token-counter", default-features = false }
 pyo3.workspace = true
 prost.workspace = true

litellm-rust/crates/python-bridge/src/lib.rs

@@ -8,6 +8,7 @@ mod execution;
 mod http;
 mod logger;
 mod marshal;
+mod model_registry;
 mod preflight;
 mod python_settings;
 mod routes;
@@ -30,6 +31,8 @@ mod _native {
     #[pymodule_export]
     use crate::logger::NativeDiagnosticProcessor;
     #[pymodule_export]
+    use crate::model_registry::bundled_model_cost_map;
+    #[pymodule_export]
     use crate::routes::audio_transcription::{atranscription, transcription};
     #[pymodule_export]
     use crate::routes::chat_completions::{acompletion, completion};
@@ -110,6 +113,7 @@ mod tests {
                 "NativeClickHouseSpendConfig",
                 "NativeClickHouseSpendStorage",
                 "trace_encode_error",
+                "bundled_model_cost_map",
                 "TokenCounter",
                 "Tokenizer",
                 "gil_stats",

litellm-rust/crates/python-bridge/src/model_registry.rs

@@ -0,0 +1,30 @@
+use pyo3::prelude::*;
+use pyo3::sync::PyOnceLock;
+use pyo3::types::PyBytes;
+
+#[pyfunction]
+pub fn bundled_model_cost_map(py: Python<'_>) -> Bound<'_, PyBytes> {
+    static BYTES: PyOnceLock<Py<PyBytes>> = PyOnceLock::new();
+    BYTES
+        .get_or_init(py, || {
+            PyBytes::new(py, litellm_model_catalog::BUNDLED_REGISTRY).unbind()
+        })
+        .bind(py)
+        .clone()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[rstest::rstest]
+    fn repeated_calls_reuse_the_bundled_registry_bytes() {
+        Python::initialize();
+        let first = Python::attach(|py| bundled_model_cost_map(py).unbind());
+        Python::attach(|py| {
+            let second = bundled_model_cost_map(py);
+            assert!(first.bind(py).is(&second));
+            assert_eq!(second.as_bytes(), litellm_model_catalog::BUNDLED_REGISTRY);
+        });
+    }
+}

litellm/__init__.py

@@ -39,6 +39,7 @@ from litellm._logging import (
     turn_on_json,
     log_level,
 )
+from litellm._logging import suppress_startup_output as suppress_startup_output
 import re
 from litellm.constants import (
     DEFAULT_BATCH_SIZE,
@@ -519,7 +520,7 @@ aiohttp_trust_env: bool = False  # set to true to use HTTP_ Proxy settings
 disable_aiohttp_transport: bool = False  # Set this to true to use httpx instead
 disable_aiohttp_trust_env: bool = False  # When False, aiohttp will respect HTTP(S)_PROXY env vars
 force_ipv4: bool = False  # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
-http2: bool = False
+http2: bool | None = None
 network_mock: bool = False  # When True, use mock transport — no real network calls
 
 ####### STOP SEQUENCE LIMIT #######
@@ -619,6 +620,8 @@ vertex_openai_models: Set = set()
 vertex_minimax_models: Set = set()
 vertex_moonshot_models: Set = set()
 vertex_zai_models: Set = set()
+vertex_qwen_models: Final[Set[str]] = set()
+vertex_ai_models: Final[Set[str]] = set()
 ai21_models: Set = set()
 ai21_chat_models: Set = set()
 nlp_cloud_models: Set = set()
@@ -630,11 +633,13 @@ fireworks_ai_models: Set = set()
 baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
+clf_ai_gateway_models: set[str] = set()  # mutable-ok: filled at import like sibling sets
 perplexity_models: Set = set()
 watsonx_models: Set = set()
 gemini_models: Set = set()
 xai_models: Set = set()
 zai_models: Set = set()
+bigmodel_models: Final[set[str]] = set()  # mutable-ok: provider registry is populated and extended at runtime
 deepseek_models: Set = set()
 tencent_models: Set = set()
 runwayml_models: Set = set()
@@ -681,6 +686,7 @@ moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
 openinfer_models: Final[Set[str]] = set()
+kyma_models: Final[set[str]] = set()  # mutable-ok: populated when the shared model catalog loads
 streamlake_models: Final[Set[str]] = set()
 zerogpu_models: Final[Set[str]] = set()
 v0_models: Set = set()
@@ -833,6 +839,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
         elif value.get("litellm_provider") == "vertex_ai-zai_models":
             key = key.replace("vertex_ai/", "")
             vertex_zai_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-qwen_models":
+            vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
+        elif value.get("litellm_provider") == "vertex_ai":
+            vertex_ai_models.add(key.removeprefix("vertex_ai/"))
         elif value.get("litellm_provider") == "ai21":
             if value.get("mode") == "chat":
                 ai21_chat_models.add(key)
@@ -850,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             bedrock_converse_models.add(key)
         elif value.get("litellm_provider") == "deepinfra":
             deepinfra_models.add(key)
+        elif value.get("litellm_provider") == "clf_ai_gateway":  # pyright: ignore[reportUnknownMemberType]  # raw cost map
+            clf_ai_gateway_models.add(key)  # pyright: ignore[reportUnknownArgumentType]  # raw cost map key
         elif value.get("litellm_provider") == "perplexity":
             perplexity_models.add(key)
         elif value.get("litellm_provider") == "watsonx":
@@ -860,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key and "fireworks-ai-default" not in key:
                 fireworks_ai_models.add(key)
+        elif value.get("litellm_provider") == "together_ai":
+            if _is_past_deprecation_date(value):
+                together_ai_models.discard(key)
+            elif not key.startswith("together-ai-"):
+                together_ai_models.add(key)
         elif value.get("litellm_provider") == "baseten":
             if _is_past_deprecation_date(value):
                 baseten_models.discard(key)
@@ -877,6 +894,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             xai_models.add(key)
         elif value.get("litellm_provider") == "zai":
             zai_models.add(key)
+        elif value.get("litellm_provider") == "bigmodel":
+            bigmodel_models.add(key)
         elif value.get("litellm_provider") == "fal_ai":
             fal_ai_models.add(key)
         elif value.get("litellm_provider") == "deepseek":
@@ -971,6 +990,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             darkbloom_models.add(key)
         elif value.get("litellm_provider") == "openinfer":
             openinfer_models.add(key)
+        elif value.get("litellm_provider") == "kyma":
+            kyma_models.add(key)
         elif value.get("litellm_provider") == "streamlake":
             streamlake_models.add(key)
         elif value.get("litellm_provider") == "zerogpu":
@@ -1097,6 +1118,7 @@ model_list = list(
     | set(ollama_models)
     | bedrock_models
     | deepinfra_models
+    | clf_ai_gateway_models
     | perplexity_models
     | set(maritalk_models)
     | runwayml_models
@@ -1107,6 +1129,7 @@ model_list = list(
     | text_completion_inception_models
     | xai_models
     | zai_models
+    | bigmodel_models
     | fal_ai_models
     | deepseek_models
     | modelscope_models
@@ -1148,6 +1171,7 @@ model_list = list(
     | publicai_models
     | darkbloom_models
     | openinfer_models
+    | kyma_models
     | streamlake_models
     | zerogpu_models
     | v0_models
@@ -1204,13 +1228,23 @@ def _build_models_by_provider() -> dict:
         | vertex_deepseek_models
         | vertex_minimax_models
         | vertex_moonshot_models
-        | vertex_zai_models,
+        | vertex_zai_models
+        | vertex_qwen_models
+        | vertex_mistral_models
+        | vertex_llama3_models
+        | vertex_ai_ai21_models
+        | vertex_openai_models
+        | vertex_ai_image_models
+        | vertex_ai_video_models
+        | vertex_embedding_models
+        | vertex_ai_models,
         "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
         "ollama": ollama_models,
         "ollama_chat": ollama_models,
         "deepinfra": deepinfra_models,
+        "clf_ai_gateway": clf_ai_gateway_models,
         "perplexity": perplexity_models,
         "maritalk": maritalk_models,
         "watsonx": watsonx_models,
@@ -1221,6 +1255,7 @@ def _build_models_by_provider() -> dict:
         "text-completion-inception": text_completion_inception_models,
         "xai": xai_models,
         "zai": zai_models,
+        "bigmodel": bigmodel_models,
         "fal_ai": fal_ai_models,
         "deepseek": deepseek_models,
         "tencent": tencent_models,
@@ -1269,6 +1304,7 @@ def _build_models_by_provider() -> dict:
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
         "openinfer": openinfer_models,
+        "kyma": kyma_models,
         "streamlake": streamlake_models,
         "zerogpu": zerogpu_models,
         "v0": v0_models,
@@ -1757,6 +1793,9 @@ if TYPE_CHECKING:
     from .llms.bespoke.decisions.transformation import (
         BespokeDecisionsConfig as BespokeDecisionsConfig,
     )
+    from .llms.eu_jev.decisions.transformation import (
+        EuJevDecisionsConfig as EuJevDecisionsConfig,
+    )
     from .llms.openai.decisions.transformation import (
         OpenAIDecisionsConfig as OpenAIDecisionsConfig,
     )
@@ -2082,6 +2121,9 @@ if TYPE_CHECKING:
         LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
     )
     from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
+    from .llms.clf_ai_gateway.chat.transformation import (
+        ClfAiGatewayConfig as _ClfAiGatewayConfig,
+    )
     from .llms.llamafile.chat.transformation import (
         LlamafileChatConfig as _LlamafileChatConfig,
     )
@@ -2111,6 +2153,7 @@ if TYPE_CHECKING:
     IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
     LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
     DeepInfraConfig: Type[_DeepInfraConfig]
+    ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
     LlamafileChatConfig: Type[_LlamafileChatConfig]
     LMStudioChatConfig: Type[_LMStudioChatConfig]
     LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]

litellm/_internal_context.py

@@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
 
 _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
 
+_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
+
 
 @contextmanager
 def internal_sub_call() -> Generator[None]:
@@ -123,6 +125,30 @@ def current_service_caller() -> str | None:
     return _service_caller.get()
 
 
+@contextmanager
+def mirrored_call() -> Generator[None]:
+    """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
+    token: Final = _mirrored_call.set(True)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
+def in_mirrored_call() -> bool:
+    return _mirrored_call.get()
+
+
+@contextmanager
+def outside_mirrored_call() -> Generator[None]:
+    """Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""
+    token: Final = _mirrored_call.set(False)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
 @contextmanager
 def pinned_billing_time(moment: datetime) -> Generator[None]:
     """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""

litellm/_lazy_imports_registry.py

@@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
     "OllamaDecisionsConfig",
     "LayaDecisionsConfig",
     "BespokeDecisionsConfig",
+    "EuJevDecisionsConfig",
     "OpenAIDecisionsConfig",
     "AzureDecisionsConfig",
     "AzureAIDecisionsConfig",
@@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
     "MistralEmbeddingConfig",
     "OpenAIImageVariationConfig",
     "DeepInfraConfig",
+    "ClfAiGatewayConfig",
     "DeepgramAudioTranscriptionConfig",
     "TopazImageVariationConfig",
     "OpenAITextCompletionConfig",
@@ -753,6 +755,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
     "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
     "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
+    "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
     "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
     "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
@@ -973,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "OpenAIImageVariationConfig",
     ),
     "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
+    "ClfAiGatewayConfig": (
+        ".llms.clf_ai_gateway.chat.transformation",
+        "ClfAiGatewayConfig",
+    ),
     "DeepgramAudioTranscriptionConfig": (
         ".llms.deepgram.audio_transcription.transformation",
         "DeepgramAudioTranscriptionConfig",

litellm/_logging.py

@@ -10,10 +10,12 @@ import sys
 from collections.abc import Iterator, Mapping
 from datetime import datetime
 from logging import Formatter
+from types import FrameType
 from typing import Final, TextIO
 from urllib.parse import unquote
 
 from pydantic import TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
@@ -26,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
 from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.secret_redaction import (
+    REDACTED,
     python_redact_string,
     python_redact_structured_value,
     redact_internal_details,
+    redact_structured_value,
 )
 from litellm.litellm_core_utils.secret_redaction import (
     redact_string as redact_secret_string,
@@ -138,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
     return redact_internal_details(value)
 
 
+_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
+_CLIENT_ITEMS: Final = TypeAdapter(list[object])
+
+
+def _redacted_client_field(key: str | None, value: object) -> object:
+    if isinstance(value, str):
+        return redact_internal_details(redact_structured_value(key, value))
+    if isinstance(value, dict):
+        return {
+            child_key: _redacted_client_field(child_key, child)
+            for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
+        }
+    if isinstance(value, (list, tuple)):
+        return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
+    if not isinstance(value, (int, float)):
+        return value
+    text: Final = str(value)
+    return value if redact_structured_value(key, text) == text else REDACTED
+
+
+def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
+    if not _ENABLE_SECRET_REDACTION:
+        return fields
+    return _redacted_client_field(None, dict(fields))
+
+
 def _substituted_color_message(record: logging.LogRecord) -> str | None:
     """Render a record's ``color_message`` against its args, or None if absent.
 
@@ -598,6 +628,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
 _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
 _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
 _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
+_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
 
 
 def _stream_is_tty(stream: TextIO | None) -> bool:
@@ -612,15 +643,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool:
         return False
 
 
-def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
-    """The plain-text log format, colorized only when both streams are an interactive terminal.
+def _color_enabled(stream: TextIO | None) -> bool:
+    """True when ANSI colors written to the stream reach an interactive terminal.
 
     Honors the NO_COLOR convention from no-color.org: color is disabled when
     NO_COLOR is present with a non-empty value.
     """
-    if os.environ.get("NO_COLOR"):
-        return _PLAIN_LOG_FORMAT
-    return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
+    return not os.environ.get("NO_COLOR") and _stream_is_tty(stream)
+
+
+def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool:
+    """True when both streams are an interactive color terminal, the one rule for log lines and startup prints."""
+    return _color_enabled(stdout) and _color_enabled(stderr)
+
+
+def terminal_text(message: str) -> str:
+    """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses."""
+    return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message)
+
+
+def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
+    """The plain-text log format, colorized only when both streams are an interactive terminal."""
+    return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT
 
 
 class LevelRoutingStreamHandler(logging.StreamHandler):
@@ -642,8 +686,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler):
         super().emit(record)
 
 
-def _parse_json_logs_env(value: str | None) -> bool:
-    """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
+def _parse_env_flag(value: str | None) -> bool:
+    """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it.
 
     Matches the reader in litellm-proxy-extras/_logging.py. The previous
     bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
@@ -656,7 +700,18 @@ def resolve_log_level(log_level: str) -> int:
     return getattr(logging, log_level.upper())
 
 
-json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
+json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS"))
+suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT"))
+
+
+class StartupOutputSettings(TypedDict, total=False):
+    json_logs: ReadOnly[object]
+    suppress_startup_output: ReadOnly[object]
+
+
+STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings)
+
+
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: Final[int] = resolve_log_level(log_level)
@@ -743,7 +798,7 @@ class JsonFormatter(Formatter):
         return dt.isoformat()
 
     def format(self, record):
-        message_str: Final = record.getMessage()
+        message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage())
         json_record: Final[dict[str, object]] = {
             "message": message_str,
             "level": record.levelname,
@@ -799,12 +854,14 @@ class CorrelationPlainFormatter(logging.Formatter):
     """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter.
 
     Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs
-    behaves the same whether or not json_logs is enabled.
+    behaves the same whether or not json_logs is enabled. Unless it was built with the color format,
+    it also drops the ANSI color codes a message carries, which a piped log would store as text.
     """
 
     def format(self, record: logging.LogRecord) -> str:
         rendered: Final = super().format(record)
-        formatted: Final = rendered if _is_redacted(record) else redact_string(rendered)
+        uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered)
+        formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored)
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:
@@ -898,29 +955,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter)
 verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
 verbose_logger.addFilter(_diagnostic_filter)
 
-_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
-
 
 def print_startup_message(message: str) -> None:
     """Write a boot-time message meant for the operator's terminal.
 
-    Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of
-    stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger
-    instead, on stdout like the print whatever handlers the logging config gave that logger, with the
-    ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log
-    level, whole and unredacted as the print showed it; a blank spacer produces no record.
+    LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them
+    off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing,
+    keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a
+    container log never stores escape codes.
+    With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one
+    INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging
+    config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at
+    the caller and emitted at any log level, whole and unredacted as the print showed it; a blank
+    spacer produces no record.
     """
+    if litellm.suppress_startup_output is True:
+        return
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO)
+
+
+def print_startup_warning(message: str) -> None:
+    """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output
+    on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on
+    stdout where the print put it, so a collector that filters on the level field keeps it."""
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING)
+
+
+def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None:
     if not litellm.json_logs:
-        print(message)  # noqa: T201  # plain mode: this is the operator-terminal print it replaces
+        import click
+
+        click.echo(terminal_text(message))
         return
     text: Final = _ANSI_ESCAPE.sub("", message).strip("\n")
     if not text:
         return
-    frame: Final = inspect.currentframe()
-    caller: Final = frame.f_back if frame is not None else None
     record: Final = verbose_proxy_logger.makeRecord(
         verbose_proxy_logger.name,
-        logging.INFO,
+        level,
         caller.f_code.co_filename if caller is not None else __file__,
         caller.f_lineno if caller is not None else 0,
         text,
@@ -934,8 +1008,8 @@ def print_startup_message(message: str) -> None:
 
 
 def _startup_json_handler() -> logging.Handler:
-    """The proxy's JSON handler on stdout, filters included, for one startup message."""
-    handler: Final = LevelRoutingStreamHandler()
+    """The proxy's JSON handler on stdout, filters included, for one startup message or warning."""
+    handler: Final = logging.StreamHandler(sys.stdout)
     handler.setFormatter(JsonFormatter())
     handler.addFilter(_secret_filter)
     handler.addFilter(_correlation_filter)

litellm/anthropic_beta_gated_fields.py

@@ -0,0 +1,325 @@
+"""Body fields that an ``anthropic-beta`` value gates.
+
+``anthropic_beta_headers_config.json`` says which beta each provider leg
+carries. When a leg drops a beta, the request fields that beta unlocks
+would make the provider 400. Under ``drop_params`` the same request degrades on
+that leg by dropping those fields too; without it the request goes out as sent
+and the provider answers its own 400, like any unsupported param. Every helper
+here is copy-on-write: the caller's containers are returned untouched when
+nothing is stripped, and a shallow copy is returned otherwise, so a fallback
+leg still sends the full request.
+"""
+
+from collections.abc import Mapping, Sequence, Set
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.types.llms.anthropic import (
+    ANTHROPIC_BETA_HEADER_VALUES,
+    ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
+    ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
+    ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
+    ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
+)
+
+ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
+ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
+
+THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
+THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
+SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
+SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
+SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
+SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
+
+_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
+
+BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,),
+        ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,),
+        ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,),
+        ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+        ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+    }
+)
+
+
+def _betas_unlocking(field: str) -> frozenset[str]:
+    return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields)
+
+
+_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
+    {
+        field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields
+    }  # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index
+)
+
+
+def field_is_gated(field: str, active_betas: Set[str]) -> bool:
+    """True when no active beta unlocks ``field``, so the field must be stripped."""
+    return _FIELD_BETAS[field].isdisjoint(active_betas)
+
+
+_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
+
+
+def _as_mapping(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MAPPING_ADAPTER.validate_python(value, strict=True)
+    except ValidationError:
+        return None
+
+
+def _as_sequence(value: object) -> tuple[object, ...] | None:
+    try:
+        return _SEQUENCE_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _is_tool_change_block(block: object) -> bool:
+    mapping: Final = _as_mapping(block)
+    block_type: Final = mapping.get("type") if mapping is not None else None
+    return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES
+
+
+def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]:
+    joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta")
+    return frozenset(piece.strip() for piece in joined.split(",") if piece.strip())
+
+
+def _validated_headers(headers: object) -> Mapping[str, str]:
+    try:
+        return _HEADERS_ADAPTER.validate_python(headers)
+    except ValidationError:
+        return MappingProxyType({})
+
+
+def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]:
+    """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched
+    case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body."""
+    body_values: Final = _as_sequence(body_betas) or ()
+    body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str))
+    return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedThinking:
+    thinking: object
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedMessages:
+    messages: Sequence[object]
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedRequest:
+    request: Mapping[str, object]
+    stripped: tuple[str, ...]
+
+
+def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking:
+    """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated."""
+    mapping: Final = _as_mapping(thinking)
+    if mapping is None or not drops_unsupported_params(drop_params):
+        return StrippedThinking(thinking=thinking, stripped=())
+    binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas)
+    display_gated: Final = mapping.get("display") == "updates" and field_is_gated(
+        THINKING_DISPLAY_UPDATES, active_betas
+    )
+    if not binding_gated and not display_gated:
+        return StrippedThinking(thinking=thinking, stripped=())
+    gated_keys: Final = frozenset(
+        {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())}
+    )
+    stripped: Final = (
+        *((THINKING_BLOCK_BINDING,) if binding_gated else ()),
+        *((THINKING_DISPLAY_UPDATES,) if display_gated else ()),
+    )
+    return StrippedThinking(
+        thinking={key: value for key, value in mapping.items() if key not in gated_keys},
+        stripped=stripped,
+    )
+
+
+def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages:
+    """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing
+    when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400."""
+    clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas)
+    output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas)
+    content: Final = message.get("content")
+    blocks: Final = _as_sequence(content) if not isinstance(content, str) else None
+    tool_changes_gated: Final = (
+        blocks is not None
+        and any(_is_tool_change_block(block) for block in blocks)
+        and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas)
+    )
+    if not clear_at_gated and not output_config_gated and not tool_changes_gated:
+        return StrippedMessages(messages=(message,), stripped=())
+    kept_content: Final[object] = (
+        [block for block in blocks if not _is_tool_change_block(block)]
+        if tool_changes_gated and blocks is not None
+        else content
+    )
+    gated_keys: Final = frozenset(
+        {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())}
+    )
+    rewritten: Final = {
+        key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys
+    }
+    stripped: Final = (
+        *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()),
+        *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()),
+        *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()),
+    )
+    if kept_content in (None, "", []) and "output_config" not in rewritten:
+        return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED))
+    return StrippedMessages(messages=(rewritten,), stripped=stripped)
+
+
+def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages:
+    mapping: Final = _as_mapping(message)
+    if mapping is None or mapping.get("role") != "system":
+        return StrippedMessages(messages=(message,), stripped=())
+    return _strip_beta_gated_system_message(mapping, active_betas)
+
+
+def _ends_on_assistant(messages: Sequence[object]) -> bool:
+    last: Final = _as_mapping(messages[-1]) if messages else None
+    return last is not None and last.get("role") == "assistant"
+
+
+def strip_beta_gated_system_messages(
+    messages: Sequence[object], active_betas: Set[str], *, drop_params: object
+) -> StrippedMessages:
+    """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same
+    sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation
+    ending on an assistant turn, which the model would read as a prefill to continue."""
+    if not drops_unsupported_params(drop_params):
+        return StrippedMessages(messages=messages, stripped=())
+    per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages)
+    stripped: Final = tuple(
+        name for result in per_message for name in result.stripped
+    )  # comprehension-ok: flattening per-message strip reports
+    if not stripped:
+        return StrippedMessages(messages=messages, stripped=())
+    kept: Final = [
+        message for result in per_message for message in result.messages
+    ]  # comprehension-ok: flattening per-message results
+    if _ends_on_assistant(kept) and not _ends_on_assistant(messages):
+        return StrippedMessages(messages=messages, stripped=())
+    return StrippedMessages(messages=kept, stripped=stripped)
+
+
+def strip_beta_gated_request_fields(
+    request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
+) -> StrippedRequest:
+    """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
+    nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
+    thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
+    messages: Final = request.get("messages")
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    stripped: Final = (*thinking.stripped, *system_messages.stripped)
+    if not stripped:
+        return StrippedRequest(request=request, stripped=())
+    rewritten: Final = {
+        **request,
+        **({"thinking": thinking.thinking} if thinking.stripped else {}),
+        **({"messages": system_messages.messages} if system_messages.stripped else {}),
+    }
+    return StrippedRequest(request=rewritten, stripped=stripped)
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedNativeRequest:
+    thinking: object
+    thinking_stripped: bool
+    messages: Sequence[object]
+    messages_stripped: bool
+    stripped: tuple[str, ...]
+
+
+def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest:
+    """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists."""
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    return StrippedNativeRequest(
+        thinking=params.get("thinking"),
+        thinking_stripped=False,
+        messages=message_sequence if message_sequence is not None else (),
+        messages_stripped=False,
+        stripped=(),
+    )
+
+
+def strip_beta_gated_native_request(
+    headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object
+) -> StrippedNativeRequest:
+    """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta``
+    list on legs that carry betas in the body, plus the body parts it may rewrite."""
+    if not drops_unsupported_params(drop_params):
+        return unstripped_native_request(optional_params, messages)
+    active_betas: Final = active_beta_values(headers, body_betas)
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params)
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    return StrippedNativeRequest(
+        thinking=thinking.thinking,
+        thinking_stripped=bool(thinking.stripped),
+        messages=system_messages.messages,
+        messages_stripped=bool(system_messages.stripped),
+        stripped=(*thinking.stripped, *system_messages.stripped),
+    )
+
+
+def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None:
+    if not stripped:
+        return
+    described: Final = ", ".join(
+        f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})"
+        if field in _FIELD_BETAS
+        else field
+        for field in sorted(set(stripped))
+    )
+    verbose_logger.warning(
+        "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on "
+        "this request, either since this provider leg does not send that anthropic-beta (see "
+        "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user "
+        "messages.",
+        described,
+        model,
+        provider,
+    )
+
+
+def strip_beta_gated_request_body(
+    request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str
+) -> Mapping[str, object]:
+    gated: Final = strip_beta_gated_request_fields(
+        request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params
+    )
+    log_beta_gated_strips(gated.stripped, model=model, provider=provider)
+    return gated.request

litellm/anthropic_beta_headers_config.json

@@ -248,6 +248,7 @@
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
+    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"

litellm/anthropic_beta_headers_manager.py

@@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str:
     return aliases.get(provider, provider)
 
 
+def has_beta_headers_config(provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    return canonical_provider in config
+
+
 def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
@@ -397,7 +404,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
-) -> tuple[dict, dict]:
+) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
     Modifies both dicts in place and returns them.

litellm/caching/Readme.md

@@ -8,10 +8,9 @@ The following caching mechanisms are supported:
 2. **RedisSemanticCache**
 3. **QdrantSemanticCache**
 4. **InMemoryCache**
-5. **DiskCache**
-6. **S3Cache**
-7. **AzureBlobCache**
-8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
+5. **S3Cache**
+6. **AzureBlobCache**
+7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
 
 ## Folder Structure
 
@@ -20,7 +19,6 @@ litellm/caching/
 ├── base_cache.py
 ├── caching.py
 ├── caching_handler.py
-├── disk_cache.py
 ├── dual_cache.py
 ├── in_memory_cache.py
 ├── qdrant_semantic_cache.py

litellm/caching/__init__.py

@@ -1,6 +1,5 @@
 from .azure_blob_cache import AzureBlobCache
 from .caching import Cache, LiteLLMCacheType
-from .disk_cache import DiskCache
 from .dual_cache import DualCache
 from .gcs_cache import GCSCache
 from .in_memory_cache import InMemoryCache

litellm/caching/_embedding_router.py

@@ -57,7 +57,7 @@ def resolve_embedding_max_input_tokens(
         return configured_max_input_tokens
     if router is None:
         return None
-    deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
+    deployment_max_input_tokens, _ = router.registry().group(embedding_model, field="token_limits")
     return deployment_max_input_tokens