scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 48

Merge remote-tracking branch 'origin/litellm_vertex_identity_labels' into litellm_vertex_passthrough_identity_labels

Browse source
This commit is contained in:
mateo-berri 2026-10-30 21:44:04 -07:00
commit 0be9cc6cc8
1225 changed files with 105960 additions and 16113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
.circleci/scripts/run_integration.sh
View file

@ -1,14 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "${GITHUB_ACTIONS:-}" = true ]; then
echo "Integration contracts are owned by CircleCI" >&2
exit 1
fi
suite="${1:?integration suite required}"
mode="${2:-standard}"
side="${3:-}"
requested_workers="${INTEGRATION_WORKERS:-}"
license_env=()
if [ -n "${LITELLM_LICENSE:-}" ]; then
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
fi
if [ "$mode" = replica ]; then
results="test-results/integration-${suite}-replica"
elif [ "$mode" = parity ]; then
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
test -f "$LITELLM_UI_PATH/index.html"
fi
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@ -107,7 +107,7 @@ sudo iptables -N integration_only
guard_created=true
sudo iptables -A integration_only -o lo -j ACCEPT
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
for service in postgres-db redis-cache; do
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
sudo iptables -A integration_only -d "$address" -j ACCEPT
done
@ -145,6 +145,9 @@ fi
if [ "$suite" = mcp ]; then
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
fi
if [ -n "$requested_workers" ]; then
export INTEGRATION_WORKERS="$requested_workers"
fi
isolated_workers=false
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
isolated_workers=true
@ -185,7 +188,7 @@ start_proxy() {
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
LITELLM_MODE=PRODUCTION \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

1
.github/CODEOWNERS vendored
View file

@ -1,5 +1,6 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai
/.github/workflows/zizmor.yml @yuneng-berri

7
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
## Screenshots / Proof of Fix
<!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
### Before (<hash>)

119
.github/scripts/assert_secret_absent.py vendored Normal file
View file

@ -0,0 +1,119 @@
import base64
import io
import os
import sys
import urllib.parse
import zipfile
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TextIO
SECRET_VARIABLE: Final = "SCANNED_SECRET"
MIN_ENCODED_LENGTH: Final = 12
MAX_ARCHIVE_DEPTH: Final = 3
MASK: Final = "***"
Forms = tuple[tuple[str, str], ...]
@dataclass(frozen=True, slots=True)
class Hit:
location: str
form: str
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
for offset in range(3):
chunk: Final = raw[offset:]
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
def secret_forms(secret: str) -> Forms:
encoded: Final = (
("url-encoded", urllib.parse.quote(secret, safe="")),
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
)
by_value: Final = MappingProxyType(
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
)
return (("raw", secret), *((name, value) for value, name in by_value.items()))
def mask(text: str, forms: Forms) -> str:
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
for form, value in forms:
if value in name:
yield Hit(mask(location, forms), f"{form} in name")
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
for form, value in forms:
if value.encode() in data:
yield Hit(mask(location, forms), form)
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
return
with zipfile.ZipFile(io.BytesIO(data)) as archive:
for member in archive.infolist():
if member.is_dir():
continue
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
for directory, subdirectories, files in os.walk(root):
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
yield from ((Path(directory, name), False) for name in files)
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
location: Final = str(path.relative_to(root))
if linked_directory:
yield Hit(mask(location, forms), "directory symlink, not scanned")
return
yield from _name_hits(location, location, forms)
yield from _content_hits(location, path.read_bytes(), forms, 0)
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
forms: Final = secret_forms(secret)
entries: Final = tuple(_walk(root))
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
return len(entries), hits
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
if len(argv) != 1:
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
return 2
secret: Final = environ.get(SECRET_VARIABLE, "")
root: Final = Path(argv[0])
if not secret:
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
return 0
if not root.is_dir():
print(f"{root} is not a directory, nothing to scan", file=out)
return 0
try:
scanned, hits = scan(root, secret)
except Exception as error:
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
return 1
for hit in hits:
print(f"secret found: {hit.location} ({hit.form})", file=out)
if hits:
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
return 1
print(f"no copy of the secret in {scanned} files under {root}", file=out)
return 0
if __name__ == "__main__":
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

9
.github/scripts/smoke_test_native_wheel.py vendored
View file

@ -7,6 +7,8 @@ import zipfile
from pathlib import Path
from typing import Final
COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json"
CHILD_SCRIPT: Final = """
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
@ -34,6 +36,9 @@ else:
after = module.gil_stats()
if not isinstance(after.get("releases"), int):
raise AssertionError(f"native module unusable after panic: {after!r}")
if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes():
raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout")
"""
@ -55,7 +60,9 @@ def main() -> int:
native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
native_path.write_bytes(archive.read(native_members[0]))
result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
result: Final = subprocess.run(
(sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False
)
if result.returncode != 0:
sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")

37
.github/workflows/cost-map-guard.yml vendored
View file

@ -8,6 +8,7 @@ on:
permissions:
contents: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@ -21,16 +22,21 @@ jobs:
with:
ref: ${{ github.event.pull_request.base.sha }}
persist-credentials: false
- name: Fetch the pull request head and its merge base
- name: Check out the base branch tip and fetch the pull request head and its merge base
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
git checkout -q --detach "$target"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
echo "target=$target" >> "$GITHUB_OUTPUT"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
@ -38,7 +44,30 @@ jobs:
- name: Run the guard
env:
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
HEAD_REF: ${{ github.event.pull_request.head.ref }}
run: |
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
- name: Check whether the pull request changes the cost map
id: changed
env:
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
echo "cost_map=false" >> "$GITHUB_OUTPUT"
else
echo "cost_map=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.changed.outputs.cost_map == 'true'
run: rustup toolchain install --no-self-update
- name: Load the cost map through model-catalog
if: steps.changed.outputs.cost_map == 'true'
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
-- "$RUNNER_TEMP/model_prices_and_context_window.json"

91
.github/workflows/cost-map-pins-recheck.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Cost map pins recheck
on:
push:
branches:
- main
- "litellm_**"
paths:
- model_prices_and_context_window.json
- litellm/model_prices_and_context_window_backup.json
- ci_cd/cost_map_pins.json
permissions: {}
jobs:
rerun-cost-map-guard:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
actions: write
contents: read
pull-requests: read
steps:
- name: Re-run the cost map guard on open pull requests the push could break
env:
GH_TOKEN: ${{ github.token }}
BASE_REF: ${{ github.ref_name }}
BEFORE_SHA: ${{ github.event.before }}
AFTER_SHA: ${{ github.event.after }}
FORCED: ${{ github.event.forced }}
COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
run: |
if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
|| ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
--jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
| grep -qx true; then
export RECHECK_PATHS="$COST_MAP_PATHS"
else
export RECHECK_PATHS="$PINS_PATHS"
fi
rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
-f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
repository(owner: $owner, name: $name) {
pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
pageInfo { hasNextPage endCursor }
}
}
}' \
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
| {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
| select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
failed=0
declare -A rerun_heads
while read -r number head_sha listed; do
[ -n "$number" ] || continue
if [ "$listed" = false ]; then
if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
echo "::error::could not list the files of #$number"
failed=1
continue
fi
[ -n "$changed" ] || continue
fi
[ -z "${rerun_heads[$head_sha]:-}" ] || continue
rerun_heads[$head_sha]=1
if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
echo "::error::could not list the cost map guard runs of #$number"
failed=1
continue
fi
if [ -z "$runs" ]; then
echo "::warning::no cost map guard run found for #$number"
continue
fi
while read -r run_id created_at; do
if [[ "$created_at" < "$rerun_cutoff" ]]; then
echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
continue
fi
gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
echo "::error::could not re-run cost map guard run $run_id for #$number"
failed=1
fi
done <<< "$runs"
done <<< "$candidates"
exit "$failed"

2
.github/workflows/image-scan.yml vendored
View file

@ -13,8 +13,10 @@ on:
- migrations/Dockerfile
- migrations/run.py
- gateway/Dockerfile
- gateway/launch.py
- gateway/main.py
- gateway/routes/allowlist.py
- litellm_cli/**
- backend/Dockerfile
- backend/main.py
- litellm-rust/**

152
.github/workflows/lens-install-smoke.yml vendored
View file

@ -1,152 +0,0 @@
name: Lens installation smoke
on:
workflow_dispatch:
inputs:
lens_image:
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
required: true
type: string
lens_upgrade_image:
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
required: true
type: string
gateway_baseline_ref:
description: Previously qualified gateway source commit, full SHA
required: true
type: string
permissions:
contents: read
concurrency:
group: lens-install-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-images:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- component: gateway
dockerfile: gateway/Dockerfile
- component: backend
dockerfile: backend/Dockerfile
- component: ui
dockerfile: ui/Dockerfile
- component: migrations
dockerfile: migrations/Dockerfile
- component: monolith
dockerfile: Dockerfile
- component: gateway
dockerfile: gateway/Dockerfile
baseline: true
- component: backend
dockerfile: backend/Dockerfile
baseline: true
- component: monolith
dockerfile: Dockerfile
baseline: true
env:
COMPONENT: ${{ matrix.component }}
DOCKERFILE: ${{ matrix.dockerfile }}
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
CURRENT_REF: ${{ github.sha }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ env.SOURCE_REF }}
- name: Verify the selected source
shell: bash -euo pipefail {0}
run: |
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
test "$SOURCE_REF" != "$CURRENT_REF"
fi
- name: Build the matching release image
run: |
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
- name: Save the matching release image
run: |
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
compression-level: 0
retention-days: 3
if-no-files-found: error
overwrite: true
helm-install:
needs: build-images
runs-on: ubuntu-latest-16-cores
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-install-*-${{ github.sha }}
merge-multiple: true
path: ${{ runner.temp }}/lens-install-images
- name: Load the matching release images
run: |
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
gzip -dc "$archive" | docker load
rm "$archive"
done
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
with:
cosign-release: v2.4.3
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ vars.LENS_READ_USER || github.actor }}
password: ${{ secrets.LENS_READ_TOKEN }}
- name: Verify independent Lens images before loading them
env:
LENS_IMAGE: ${{ inputs.lens_image }}
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
shell: bash -euo pipefail {0}
run: |
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
docker pull "$image"
done
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
- name: Install pinned Kubernetes test tools
run: |
curl --fail --location --output "$RUNNER_TEMP/kind" \
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
chmod +x "$RUNNER_TEMP/kind"
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
chmod +x "$RUNNER_TEMP/kubectl"
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
- name: Install, ingest, upgrade, and restart both charts
run: bash tests/e2e/migrations/lens_helm_smoke.sh

6
.github/workflows/spend-logs-guard.yml vendored
View file

@ -19,17 +19,17 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.base.sha }}
ref: ${{ github.event.pull_request.base.ref }}
persist-credentials: false
- name: Fetch the pull request head, its merge base and its reviews
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

199
.github/workflows/test-integration.yml vendored Normal file
View file

@ -0,0 +1,199 @@
name: integration
on:
pull_request:
types: [labeled]
schedule:
- cron: "5 8-20/3 * * 1-6"
timezone: America/Los_Angeles
workflow_dispatch:
inputs:
suite:
description: Integration group to run
type: choice
options:
- all
- providers
- extensions
- accounting
- management
- management-replica
- database
- database-replica
- mcp
- security
- cost
- sdk
- browser
default: all
workers:
description: Workers per group, blank keeps each group's default
type: string
default: ""
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integration:
name: integration ${{ matrix.suite }}
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
environment: integration-license
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: circle_test
ports:
- 5432:5432
options: >-
--shm-size 1g
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 20
redis:
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
timeout-minutes: 5
uses: ./.github/actions/cache-prisma-binaries
- name: Install dependencies
timeout-minutes: 20
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Install redis-server 7.2.16
timeout-minutes: 3
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Install MCP SDK1 peer
if: matrix.suite == 'mcp'
timeout-minutes: 5
run: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
- name: Set up Node.js
if: matrix.suite == 'browser'
timeout-minutes: 5
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: |
ui/litellm-dashboard/package-lock.json
tests/e2e/ui/package-lock.json
- name: Install browser dependencies and build the dashboard
if: matrix.suite == 'browser'
timeout-minutes: 20
run: |
(cd ui/litellm-dashboard && npm ci)
(cd tests/e2e/ui && npm ci)
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
- name: Run integration contracts
timeout-minutes: 45
env:
SUITE: ${{ matrix.suite }}
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
run: |
case "$SUITE" in
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
esac
- name: Verify every case ran exactly once
if: ${{ !cancelled() && matrix.suite != 'browser' }}
timeout-minutes: 2
run: |
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
test -n "$record"
mkdir -p "$RUNNER_TEMP/shards/0"
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
- name: Report MCP coverage
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
timeout-minutes: 5
run: |
coverage_dir=test-results/integration-mcp/coverage
test -f "$coverage_dir/coverage.txt" && exit 0
export COVERAGE_FILE="$PWD/$coverage_dir/data"
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
tail -n 1 "$coverage_dir/coverage.txt"
- name: Check results for the license
id: secret-scan
if: ${{ !cancelled() }}
timeout-minutes: 5
env:
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
run: python .github/scripts/assert_secret_absent.py test-results
- name: Upload results
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
timeout-minutes: 10
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
path: test-results/
retention-days: 14

29
.github/workflows/test-linting.yml vendored
View file

@ -48,11 +48,13 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$MERGE_BASE"
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@ -264,6 +266,7 @@ jobs:
name: ui
permissions:
contents: read
pull-requests: read
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
@ -281,16 +284,12 @@ jobs:
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# base.sha is the base branch tip from when the PR was opened, while
# actions/checkout leaves HEAD on a merge of the PR into the *current*
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
# landed since, so a PR that touches no UI file still gets linted
# against hundreds of other people's files. Diff the PR head against its
# own merge base instead, which is exactly what this PR changed.
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
@ -376,10 +375,10 @@ jobs:
exit 0
fi
files="$(git diff --name-only "$base" HEAD)"
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "No proxy, types or generator changes in this pull request, nothing to verify."
echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
echo "relevant=false" >> "$GITHUB_OUTPUT"
fi
@ -630,6 +629,12 @@ jobs:
- name: check_migrations_lock_timeout
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
- name: check_no_none_string_error_fields
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
- name: test_check_no_none_string_error_fields
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py

1
.github/workflows/test-terraform-provider.yml vendored
View file

@ -12,6 +12,7 @@ on:
paths:
- "terraform/provider/**"
- "litellm/proxy/**"
- "enterprise/**"
- ".github/workflows/test-terraform-provider.yml"
permissions:

20
.github/workflows/test-unit.yml vendored
View file

@ -227,6 +227,7 @@ jobs:
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
tests/unit/llms/test_predibase_transformation.py
tests/unit/llms/test_subscription_accounts.py
tests/unit/llms/a2a
tests/unit/llms/aiml
tests/unit/llms/aiohttp_openai
@ -236,6 +237,7 @@ jobs:
tests/unit/llms/azure_ai
tests/unit/llms/base_llm
tests/unit/llms/baseten
tests/unit/llms/bigmodel
tests/unit/llms/black_forest_labs
tests/unit/llms/brave
tests/unit/llms/bytez
@ -244,6 +246,7 @@ jobs:
tests/unit/llms/chatgpt
tests/unit/llms/clarifai
tests/unit/llms/claude_code
tests/unit/llms/clf_ai_gateway
tests/unit/llms/cloudflare
tests/unit/llms/codestral
tests/unit/llms/codex
@ -321,6 +324,7 @@ jobs:
tests/unit/llms/openai_like
tests/unit/llms/opencode
tests/unit/llms/opensandbox
tests/unit/llms/opensearch
tests/unit/llms/openrouter
tests/unit/llms/ovhcloud
tests/unit/llms/parallel_ai
@ -1001,9 +1005,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1148,18 +1152,20 @@ jobs:
env:
CI: "true"
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
if [ -z "$BASE_SHA" ]; then
if [ -z "$PR_NUMBER" ]; then
echo "Push to $GITHUB_REF_NAME: running the full suite"
full_suite
exit 0
fi
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()
@ -1449,9 +1455,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

9
.grype.yaml
View file

@ -11,3 +11,12 @@ ignore:
package:
name: zlib
type: apk
- vulnerability: GHSA-v4x8-3wr5-3wq8
reason: >-
filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
and the image then moves off 3.32.7, which retires this version-scoped rule
package:
name: filelock
version: 3.32.7
type: python

7
AGENTS.md
View file

@ -31,6 +31,8 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
Before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging, read the architecture decision records in `docs/adrs/`, which say why those layers are shaped the way they are
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
@ -39,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis
@ -105,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- No file sprawl: deliberate file and folder structure
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
Follow conventional commits for commit names and PR titles

2
ARCHITECTURE.md
View file

@ -2,6 +2,8 @@
This document helps contributors understand where to make changes in LiteLLM.
For why a layer is shaped the way it is, read the architecture decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) (Python SDK and proxy) and [`litellm-rust/docs/adrs/`](litellm-rust/docs/adrs/AGENTS.md) (Rust core)
---
## How It Works

36
CONTRIBUTING.md
View file

@ -8,6 +8,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
- [ ] **Prove it works end to end** - If your change affects what the proxy or SDK does, paste the commands you ran against a live proxy and their output in the PR - [see details](#proving-your-change-works-end-to-end)
#### Proxy (Backend) PRs
@ -248,6 +249,41 @@ If `make test-unit` fails:
- **Test edge cases**: Don't just test the happy path
- **Update documentation**: If you change APIs, update docs
## Proving Your Change Works End to End
Tests are required, but they do not prove that your change works for a user. A PR that changes what the proxy or SDK does shows a real run in its "Screenshots / Proof of Fix" section, meaning the exact commands you sent to a live proxy and the output that came back, against real provider APIs and not mocks. `pytest` output does not count as proof, because a passing test only shows that the code does what its mocks expect. A PR that changes only docs or CI shows the rendered page or the CI run instead
A good proof looks like this:
1. Install the proxy dependencies with `make install-proxy-dev`, then start the proxy from your branch with a config that has the model you changed, `uv run litellm --config your_config.yaml --detailed_debug`
2. Send the request a user would send with `curl`, and paste both the command and the response. When cost or routing is involved, include the response headers, since `x-litellm-response-cost` is what the user sees
3. For a bug fix, do this twice, once at the merge base with BerriAI/litellm's main (`git merge-base origin/main HEAD`, with your remote for BerriAI/litellm in place of `origin` in a fork clone) to show the bug and once at your branch's latest commit to show the fix. Name both commit hashes
4. If your change is reachable from more than one endpoint (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`), show each one
5. If the request can stream, show a `"stream": true` run too, because streaming builds usage and cost on a different path. A stream sends its headers before its usage exists, so show the `usage.cost` on the last chunk, which `/v1/chat/completions` sends when the request has `"stream_options": {"include_usage": true}` and other endpoints send with `include_cost_in_streaming_usage: true` under `litellm_settings`, or, when the proxy has a database, the request's row on the Admin UI's Logs page or `GET /spend/logs`
6. For UI changes, add before and after screenshots and name the page
7. If users reach your change through a coding tool such as Claude Code or Codex, drive that tool the way a user does and add before and after screenshots of it, because a `curl` or a headless run alone does not show what the user sees
```bash
curl -sD - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{"model": "your-model", "messages": [{"role": "user", "content": "hi"}]}'
```
When the provider reports its own usage or spend numbers, compare the proxy's numbers against them in the proof
## Contributing an LLM Provider Integration
Provider work (a new provider, a new parameter, a new usage or cost field) is the most common kind of contribution, and it is where PRs get reworked most often. Follow these rules
**Find the existing machinery before you add plumbing.** LiteLLM already has generic paths that move data between the endpoints, the provider transformations, and cost tracking. A second, hand-rolled path for the same data is a common reason a provider PR gets rewritten before merge. [`ARCHITECTURE.md`](ARCHITECTURE.md) shows where the layers live, and the decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) explain why they are shaped that way. For example, [the ADR on provider usage fields](docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md) covers how provider-specific usage fields reach cost tracking. If you cannot find a mechanism for what you need, say so in the PR or ask in [#pr-review on Slack](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) before you build one
**Bill a new provider usage field from the usage object.** Leave the field on the usage object as an extra and read it in that provider's `cost_calculator.py`. A provider without one gets a new `cost_calculator.py` and an arm for it in `cost_per_token` (`litellm/cost_calculator.py`), which picks the calculator by provider. If the provider's transform builds `Usage` from named fields, as the Anthropic, Gemini, and Bedrock Converse ones do, pass the new field there too. Streamed requests rebuild usage in `ChunkProcessor.calculate_usage` (`litellm/litellm_core_utils/streaming_chunk_builder_utils.py`), which does not copy extras yet the way the Responses-to-chat bridge does (minus a short exclusion list), so until it does, bill the field on streams by naming it there, as Anthropic's `inference_geo` is. A provider that reports its own charge for the request, as OpenRouter does in `usage.cost`, is billed at that charge from `_hidden_params` before any calculator runs, unless the deployment sets its own prices. When the field counts a built-in tool such as web search, add the provider to `get_cost_for_web_search_request` in `litellm/llms/__init__.py`, and add a gate to `litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py` only when no existing gate fires. Then show the field's cost in your proof on every endpoint it reaches, streaming included
**Do not change what a caller sees to make internals easier.** Each endpoint keeps the schema of the API it emulates. A `/v1/responses` caller gets `input_tokens` and a `/v1/chat/completions` caller gets `prompt_tokens`, whatever the provider sent and whatever cost tracking needs. A provider usage field that shows up as an extra key is fine, since it only adds a key, but renaming, removing, or reshaping a key the endpoint defines is not. Reshaping a public response to feed an internal consumer breaks every user of that endpoint
If your change makes an architecture decision, such as a new cross-cutting mechanism or a deliberate difference from a provider's API shape, add an ADR to [`docs/adrs/`](docs/adrs/AGENTS.md) in the same PR
## Building and Running Locally
### LiteLLM Proxy Server

2
Makefile
View file

@ -153,7 +153,7 @@ lint-install:
# the same set CI's ':(glob)litellm/**/*.py' selects.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@base_ref=$$($(RESOLVE_BASE)) && \
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \

52
README.md
View file

@ -141,12 +141,56 @@ response = client.chat.completions.create(
)
```
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
</details>
<details>
<summary><b>Auto Routing</b> - Choose a model for each prompt (Python SDK + AI Gateway)</summary>
### Python SDK
[**Docs: Autorouter SDK**](https://docs.litellm.ai/docs/autorouter_sdk)
Install with `uv add litellm` and set `OPENAI_API_KEY` for the models below.
```python
import asyncio
from litellm import Router
router = Router(model_list=[
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-6-luna"}},
{"model_name": "capable", "litellm_params": {"model": "openai/gpt-6.1-sol"}},
{"model_name": "auto", "litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "heuristic",
"tiers": {"SIMPLE": "fast", "MEDIUM": "fast",
"COMPLEX": "capable", "REASONING": "capable"},
"return_raw_model_name": True,
},
"complexity_router_default_model": "capable",
}},
])
response = asyncio.run(router.acompletion(
model="auto", messages=[{"role": "user", "content": "What is 2+2?"}],
))
print(response.model, response.choices[0].message.content)
```
### AI Gateway (Proxy Server)
[**Create an Auto Router**](https://docs.litellm.ai/docs/auto_router/setup), then call its model name. Replace `auto` and the URL below with yours, and set `LITELLM_API_KEY` to your virtual key.
```shell
curl http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "Content-Type: application/json" \
-d '{"model":"auto","messages":[{"role":"user","content":"What is 2+2?"}]}'
```
</details>
<details>
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
<summary><b>A2A</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
@ -335,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
@ -354,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
@ -442,7 +488,7 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/python_sdk">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>

1
backend/Dockerfile
View file

@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \

1
backend/routes/allowlist.py
View file

@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/subscription_accounts/",
"/cost_optimization/",
"/cost/",
"/credentials",

147
ci_cd/cost_map_guard.py
View file

@ -1,37 +1,67 @@
"""Guard the cost map on pull requests.
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files and may only add or update models.
Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
file checks: the files parse, the backup copy matches the root file, and every capability pinned in
ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
Bot) get the human checks and may not touch the pins file either.
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
PINS_PATH: Final = "ci_cd/cost_map_pins.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
BOT_AUTHOR_TYPE: Final = "Bot"
MERGE_CONFLICT_EXIT_CODE: Final = 1
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
CostMap = dict[str, object]
_MISSING: Final = object()
@dataclass(frozen=True, slots=True)
class Snapshot:
cost_map: str
backup: str
pins: str
@dataclass(frozen=True, slots=True)
class MergeFailed:
stderr: str
@dataclass(frozen=True, slots=True)
class Pin:
key: str
field: str
value: object
checked: str
source: str
def _parse_object(text: str, path: str) -> CostMap | str:
@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
def _pin(key: str, field: str, spec: object) -> Pin | str:
malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
if not isinstance(spec, dict) or "value" not in spec:
return malformed
checked: Final = spec.get("checked")
source: Final = spec.get("source")
if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
return malformed
return Pin(key, field, spec["value"], checked, source)
def _pins(parsed: CostMap) -> Iterator[Pin | str]:
for key, fields in parsed.items():
if not isinstance(fields, dict):
yield f"{PINS_PATH}: {key} must map field names to pins"
continue
for field, spec in fields.items():
yield _pin(key, field, spec)
def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
for pin in _pins(parsed):
if isinstance(pin, str):
yield pin
elif pin.key not in entries:
yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
yield (
f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
)
def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
if not pins_text:
return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
parsed: Final = _parse_object(pins_text, PINS_PATH)
if isinstance(parsed, str):
return (parsed,)
return tuple(_pin_violations(parsed, _entries(head_map)))
def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
if isinstance(base_map, str):
@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
)
def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
if PINS_PATH not in changed_files:
return ()
if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
if author_type == BOT_AUTHOR_TYPE:
return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
return ()
def touches_cost_map(changed_files: Sequence[str]) -> bool:
return any(path in GUARDED_PATHS for path in changed_files)
return any(path in CHECKED_PATHS for path in changed_files)
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
if merged is None:
return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
if not merged.pins and not base.pins:
return ()
merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
if isinstance(merged_map, str):
return (f"after merging into the base branch, {merged_map}",)
return pin_failures(merged.pins, merged_map)
def guard_failures(
base: Snapshot,
head: Snapshot,
merged: Snapshot | None,
changed_files: Sequence[str],
bot: bool,
*,
@ -147,6 +241,7 @@ def guard_failures(
*_file_failures(
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
),
*_merged_pin_failures(base, merged),
*(_bot_failures(base, head_map, changed_files) if bot else ()),
)
@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
def snapshot(revision: str) -> Snapshot:
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
result: Final = subprocess.run(
("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
check=False,
capture_output=True,
text=True,
)
tree: Final = result.stdout.split("\n", 1)[0]
if result.returncode == 0:
return tree
if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
return None
return MergeFailed(result.stderr.strip())
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
parser: Final = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base", required=True, help="merge base of the pull request")
parser.add_argument("--head", required=True, help="head commit of the pull request")
env_target: Final = os.environ.get(TARGET_ENV)
parser.add_argument(
"--target",
default=env_target,
required=env_target is None,
help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
)
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
args: Final = parser.parse_args(argv)
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
if changed is None:
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
return 1
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
if not bot and not touches_cost_map(changed):
print(f"cost map guard passed ({contract_for(bot, changed)})")
return 0
merged: Final = merged_tree(args.base, args.target, args.head)
if isinstance(merged, MergeFailed):
print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
return 1
failures: Final = (
*guard_failures(
snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
),
*pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
)
contract: Final = contract_for(bot, changed)
if failures:
print(f"cost map guard failed ({contract}):")

16
ci_cd/cost_map_pins.json Normal file
View file

@ -0,0 +1,16 @@
{
"fireworks_ai/accounts/fireworks/models/minimax-m3": {
"supports_vision": {
"value": true,
"checked": "2026-09-26",
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
}
},
"fireworks_ai/minimax-m3": {
"supports_vision": {
"value": true,
"checked": "2026-09-26",
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
}
}
}

11
codecov.yaml
View file

@ -15,9 +15,10 @@ ignore:
flag_management:
default_rules:
carryforward: true
# Dead flags no CI job uploads anymore: their carried-forward sessions were
# measured against old revisions, and the stale line maps mark comment lines
# of since-edited files as missed, sinking patch coverage on unrelated PRs.
# Dead flags no CI job uploads anymore, and flags whose job is path-gated on
# PRs: their carried-forward sessions were measured against old revisions, and
# the stale line maps mark comment lines of since-edited files as missed,
# sinking patch coverage on unrelated PRs.
individual_flags:
- name: proxy-mgmt-behavior
carryforward: false
@ -99,6 +100,10 @@ flag_management:
carryforward: false
- name: proxy-db-endpoints-and-responses
carryforward: false
- name: roi-postgres
carryforward: false
- name: redis-compat
carryforward: false
component_management:
individual_components:

5
docs/adrs/AGENTS.md Normal file
View file

@ -0,0 +1,5 @@
# Architecture decision records for the Python SDK and proxy
This folder records decisions about the Python SDK and proxy the way [`litellm-rust/docs/adrs/`](../../litellm-rust/docs/adrs/AGENTS.md) records them for the Rust core, and that file's process applies here unchanged: one short `adr_NNN_short_title.md` per decision that is hard to reverse or that a new contributor would reasonably question, with Context, Decision, Alternatives Considered, and Consequences, superseded by a new ADR instead of edited once accepted. Each folder numbers its own ADRs, so a reference to an ADR in the other folder names its path
Read the ADRs here before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging. A PR that makes a decision of that kind adds its ADR in the same PR

39
docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md Normal file
View file

@ -0,0 +1,39 @@
# ADR 001: Provider usage fields ride as extras on the normalized usage object
Status: Accepted
## Context
Providers report usage fields that no other provider has, such as xAI's `usage.server_side_tool_usage_details.web_search_calls` or Anthropic's `usage.inference_geo`, and cost tracking has to read them or that spend never reaches `LiteLLM_SpendLogs`. One provider response can be served on `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, streaming or not, so a field that only one of those paths carries is billed wrong on the others
[#30817](https://github.com/BerriAI/litellm/pull/30817) fixed unbilled xAI web searches. One draft did the copy in the xAI Responses transform to keep provider code out of shared files, and that swapped `/v1/responses` usage to the chat shape, so callers of that endpoint lost the Responses schema. The merged version keeps the schema, copies unknown usage fields in the shared Responses-to-chat bridge for every provider, and gates on the reported field without naming the provider
In October 2026, `Usage` and `ResponseAPIUsage` kept unknown usage keys as pydantic extras, and that bridge copied them. Paths that build usage from named fields kept only the fields they name: the Anthropic, Gemini, and Bedrock Converse transforms, the stream builder, and the bridges from chat usage to `/v1/responses` and `/v1/messages`
## Decision
Provider-only usage fields stay on the usage object as pydantic extras, and cost tracking reads them there, in that provider's cost calculator or in a shared gate that checks the field instead of the provider. An extra reaches callers as an added usage key. No transform renames, removes, or reshapes a usage field the endpoint's schema defines to feed cost tracking, though it may fill one LiteLLM's usage object already defines, such as `server_tool_use`
A shared path that rebuilds usage, such as the stream builder or a bridge between endpoint shapes, copies the extras instead of naming provider fields. A provider transform that builds usage from named fields names the new field. Neither is bypassed with a side channel. Older code that differs, such as the Azure branches in the shared tracker and the Gemini pricing tier read from `_hidden_params`, predates this ADR
## Alternatives Considered
**Reshape usage in the provider transform**, as a draft of #30817 did. It changes the schema that callers of that endpoint rely on, and every provider and endpoint needs its own copy
**A typed field on `ResponseAPIUsage` and `Usage` for each provider's quirk**. The types grow without bound and every bridge needs an update for each one. A field several providers report, like `server_tool_use`, does get one
**A side channel such as `_hidden_params`**. Cost calculators receive `Usage`, so a side channel makes a second source of truth for the same number
**Provider branches or provider field names in shared code**. They put one provider's logic in a file every provider shares, which is why #30817 replaced its xAI-only gate with one that checks the reported field
## Consequences
Benefits:
- Billing a new provider usage field touches that provider's calculator (plus a `cost_per_token` arm when the calculator is new), its transform when that names fields, and for a built-in tool at most one gate and one `get_cost_for_web_search_request` arm
Costs:
- Extras are untyped, so a reader validates them before use, as `_usage_reports_server_side_web_search_calls` does
- The stream builder and the bridges from chat usage did not copy extras when this was written, so until they do, a field billed on those paths is named in them, and tests and proof runs cover each endpoint and `"stream": true`
- This covers usage fields LiteLLM prices, not request-side params or a provider's own charge for the request, which OpenRouter and opencode Go report and `response_cost_calculator` reads from `_hidden_params` first

10
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -15,7 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.proxy.guardrails._content_utils import (
is_non_conversational_request,
iter_message_text,
)
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
async def async_moderation_hook(
self,
data: dict,
data: dict[str, object],
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if is_non_conversational_request(call_type, data):
verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
return
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:

3
enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
View file

@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
try:
disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
if disabled_callbacks is not None:
verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
#########################################################
# premium user check
#########################################################

19
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -398,7 +398,7 @@ class CheckBatchCost:
deployment. Once that deployment leaves the router, default fallbacks can
silently send the retrieve to a provider that never saw the batch, so its
404 must not retire a potentially recoverable row."""
return self.llm_router.get_deployment(model_id=model_id) is not None
return self.llm_router.registry().deployment(model_id) is not None
@staticmethod
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
@ -564,7 +564,7 @@ class CheckBatchCost:
def _get_deployment_id_for_bare_model(
self, bare_model_name: str, llm_provider: str
) -> Optional[str]:
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
deployment_id = (
self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
)
@ -580,7 +580,7 @@ class CheckBatchCost:
) -> Optional[str]:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment in self.llm_router.get_model_list(model_name=None) or []:
for deployment in self.llm_router.registry().deployments() or []:
litellm_params = deployment.get("litellm_params") or {}
actual_model = litellm_params.get("model")
if not isinstance(actual_model, str):
@ -623,8 +623,9 @@ class CheckBatchCost:
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
registry = self.llm_router.registry()
for deployment_id in registry.names(model_group, kind="ids"):
deployment_info = registry.deployment(deployment_id)
if deployment_info is None:
continue
try:
@ -743,7 +744,7 @@ class CheckBatchCost:
except (IndexError, AttributeError):
pass
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
@ -766,7 +767,7 @@ class CheckBatchCost:
except Exception:
pass
deployment_info = self.llm_router.get_deployment(model_id=model_id)
deployment_info = self.llm_router.registry().deployment(model_id)
if deployment_info is None:
verbose_proxy_logger.info(
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
@ -996,7 +997,7 @@ class CheckBatchCost:
)
try:
deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
deployment_info: Final = self.llm_router.registry().deployment(model_id)
if deployment_info is None:
verbose_proxy_logger.error(
f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
@ -1009,7 +1010,7 @@ class CheckBatchCost:
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
cost_provider: Final = cast(BatchFileProvider, llm_provider)
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
file_access: Final = {
**credentials,
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),

18
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
from typing import TYPE_CHECKING, Final
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
@ -273,7 +273,7 @@ class CheckResponsesCost:
def _resolve_deployment(self, response_id: str) -> Deployment | None:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
return self.llm_router.registry().deployment(model_id) if model_id is not None else None
async def _get_response(
self,
@ -290,12 +290,14 @@ class CheckResponsesCost:
sees provider env vars, so it fails for every deployment whose credentials
live in the config; the row then never leaves ``queued``.
"""
if not via_router:
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
router_response = await self.llm_router.aget_responses(
response_id=response_id, litellm_metadata=litellm_metadata
retrieved: Final[object] = (
await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
if via_router
else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
)
return cast(ResponsesAPIResponse, router_response)
if not isinstance(retrieved, ResponsesAPIResponse):
raise TypeError(f"Retrieving {response_id} did not return a response object")
return retrieved
async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
return await self._get_response(
@ -413,7 +415,7 @@ class CheckResponsesCost:
if target.deployment is None or not isinstance(requested, str):
return MappingProxyType({})
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
self.llm_router.registry().deployments(requested, team_id=team_id) or ()
)
serving_id: Final = target.deployment.model_info.id
return MappingProxyType(

6
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
if llm_router is None or model_id is None:
return None
try:
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
except Exception as error:
verbose_logger.warning(
"Failed to retrieve credentials for provider file "
@ -2650,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
credentials = llm_router.registry().credentials(model_id, form="with_provider")
delete_data = {
**filtered_data,
**(
@ -2694,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# against the deployment's configured bucket, which they only
# trust from this immutable server-side snapshot, never from
# request params.
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
credentials = llm_router.registry().credentials(model_id, form="with_provider")
if credentials is not None:
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
else:

2
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -290,7 +290,7 @@ def _check_team_project_limits(
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
def _project_models_expanding_at_request_time(

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.80"
version = "0.1.81"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.80"
version = "0.1.81"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

63
examples/bigmodel.md Normal file
View file

@ -0,0 +1,63 @@
# BigModel (Zhipu AI)
Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
```yaml
model_list:
- model_name: glm
litellm_params:
model: bigmodel/glm-5.3
api_key: os.environ/BIGMODEL_API_KEY
api_base: https://open.bigmodel.cn
```
One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
| Proxy endpoint | BigModel endpoint |
| --- | --- |
| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
| `/v1/responses` | `/api/v1/responses` |
| `/v1/messages` | `/api/anthropic/v1/messages` |
`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
```python
import litellm
chat = litellm.completion(
model="bigmodel/glm-5.3",
messages=[{"role": "user", "content": "Hello"}],
thinking={"type": "enabled"},
reasoning_effort="low",
)
response = litellm.responses(
model="bigmodel/glm-5.3",
input="Hello",
reasoning={"effort": "low"},
store=True,
)
async def messages():
return await litellm.anthropic.messages.acreate(
model="bigmodel/glm-5.3",
max_tokens=1024,
messages=[{"role": "user", "content": "Hello"}],
thinking={"type": "adaptive"},
output_config={"effort": "low"},
)
```
Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

2
gateway/Dockerfile
View file

@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
@ -144,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma

44
gateway/launch.py
View file

@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
wins under token auth too, so exporting it here is enough for every worker to
pick the pooled URL up unchanged.
LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
before it can answer its first 5 s health ping, so a slow host would kill each one.
Workers then load the gateway app through ``worker_app`` without garbage collection
pauses, which hold the GIL long enough to miss a ping.
Run with:
python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
"""
@ -15,19 +20,19 @@ Run with:
import os
import sys
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from typing import TYPE_CHECKING, Final
from uvicorn.main import main as uvicorn_main
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import (
PgBouncerError,
PgBouncerSettings,
export_pooled_database_url,
start_in_container_pgbouncer,
)
from litellm_cli.main import without_gc_pauses
GATEWAY_APP: Final = "gateway.main:app"
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
return (GATEWAY_APP, *argv)
return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
return (GATEWAY_APP, "--factory", *argv)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
def pool_database_url(
settings: DatabaseURLSettings,
pgbouncer: PgBouncerSettings,
settings: "DatabaseURLSettings",
pgbouncer: "PgBouncerSettings",
environ: Mapping[str, str],
) -> str | PgBouncerError | None:
) -> "str | PgBouncerError | None":
"""Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
The upstream URL is whatever ``apply_to_env`` assembled from the discrete
``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
auth the pooler mints and renews the upstream token itself.
"""
from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
if not pgbouncer.enabled:
return None
upstream_url: Final = environ.get("DATABASE_URL")
@ -58,11 +65,20 @@ def pool_database_url(
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
def worker_app() -> "FastAPI":
with without_gc_pauses():
from gateway.main import app
return app
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
settings: Final = DatabaseURLSettings.from_env()
settings.apply_to_env()
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)

1
gateway/routes/allowlist.py
View file

@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
"/test",
"/debug/memory/summary",
"/api/event_logging/batch",
"/eu_jev/v1/systemone",
}
)

7
helm/litellm-helm/Chart.lock
View file

@ -5,8 +5,5 @@ dependencies:
- name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: 18.19.1
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
generated: '2026-10-09T04:18:10.643228000Z'
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
generated: '2026-10-30T02:13:11.505726000Z'

3
helm/litellm-helm/Chart.yaml
View file

@ -39,6 +39,3 @@ dependencies:
version: "18.19.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

8
helm/litellm-helm/README.md
View file

@ -256,6 +256,10 @@ itself.
## Connect Lens
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
## Shutdown accounting
`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`

BIN
helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz
View file

Binary file not shown.

97
helm/litellm-helm/templates/_helpers.tpl
View file

@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -402,95 +398,28 @@ shutdown drain window.
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

12
helm/litellm-helm/templates/ingress.yaml
View file

@ -1,8 +1,6 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "litellm.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- $lensMode := include "litellm.lens.mode" . -}}
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@ -46,18 +44,18 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
name: {{ $.Values.lens.service.name }}
port:
number: {{ $.Values.lensWorker.service.port }}
number: {{ $.Values.lens.service.port }}
{{- else }}
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
servicePort: {{ $.Values.lensWorker.service.port }}
serviceName: {{ $.Values.lens.service.name }}
servicePort: {{ $.Values.lens.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}

3
helm/litellm-helm/templates/lens/clickhouse.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

3
helm/litellm-helm/templates/lens/deployment.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

3
helm/litellm-helm/templates/lens/ingress.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

3
helm/litellm-helm/templates/lens/secrets.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

3
helm/litellm-helm/templates/lens/service.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

128
helm/litellm-helm/tests/lens_connection_tests.yaml Normal file
View file

@ -0,0 +1,128 @@
suite: lens connection
templates:
- deployment.yaml
- configmap-litellm.yaml
- ingress.yaml
tests:
- it: connects the proxy to an external Lens
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service ahead of the host paths
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- equal:
path: spec.rules[0].http.paths[0].path
value: /lens-ingest
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

84
helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml
View file

@ -1,84 +0,0 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

187
helm/litellm-helm/tests/lens_modes_tests.yaml
View file

@ -1,187 +0,0 @@
---
suite: Lens deployment modes
templates:
- ingress.yaml
- configmap-litellm.yaml
- deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

52
helm/litellm-helm/tests/lens_modes_validation_tests.yaml
View file

@ -1,52 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- configmap-litellm.yaml
- deployment.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

59
helm/litellm-helm/tests/lens_saved_secrets_tests.yaml
View file

@ -1,59 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

167
helm/litellm-helm/tests/lens_service_tests.yaml
View file

@ -1,167 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- configmap-litellm.yaml
- deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects deployment.yaml to the shared Lens service
template: deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.hosts:
- host: gateway.example
paths:
- path: /
pathType: Prefix
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference

162
helm/litellm-helm/tests/lens_setup_tests.yaml
View file

@ -1,162 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

69
helm/litellm-helm/values.yaml
View file

@ -668,70 +668,15 @@ serviceMonitor:
# - test-namespace
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

6
helm/litellm/Chart.lock
View file

@ -1,6 +0,0 @@
dependencies:
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
generated: '2026-10-09T04:18:10.642706000Z'

5
helm/litellm/Chart.yaml
View file

@ -6,8 +6,3 @@ version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
dependencies:
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

6
helm/litellm/README.md
View file

@ -1,10 +1,8 @@
# LiteLLM componentized chart
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
This chart deploys separate gateway, backend and UI services
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.

BIN
helm/litellm/charts/lens-0.1.0-dev.0.tgz
View file

Binary file not shown.

96
helm/litellm/templates/_helpers.tpl
View file

@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
collector containers through an emptyDir. Empty when the sidecar is off
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
*/}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -635,93 +631,27 @@ shutdown drain window.
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

12
helm/litellm/templates/ingress.yaml
View file

@ -166,14 +166,14 @@ spec:
- host: {{ $componentHosts.gateway | quote }}
http:
paths:
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
- path: /
pathType: Prefix
@ -249,15 +249,15 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---

3
helm/litellm/templates/lens/clickhouse.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

3
helm/litellm/templates/lens/deployment.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

3
helm/litellm/templates/lens/ingress.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

3
helm/litellm/templates/lens/secrets.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

3
helm/litellm/templates/lens/service.yaml
View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

13
helm/litellm/tests/component_hosts_tests.yaml
View file

@ -15,7 +15,6 @@ tests:
ingress.componentHosts.ui: ui.example.com
ingress.componentHosts.backend: backend.example.com
ingress.componentHosts.gateway: gateway.example.com
lensWorker.mode: disabled
ingress.tls:
- hosts:
- ui.example.com
@ -83,12 +82,12 @@ tests:
ui.backendUrl: https://backend.example.com
ui.publicUrl: https://ui.example.com
backend.gatewayUrl: https://gateway.example.com
lensWorker.mode: external
lensWorker.externalServiceName: existing-lens
lensWorker.externalUrl: https://lens.example.com
lensWorker.publicUrl: https://lens.example.com
lensWorker.serviceTokenSecret.name: existing-lens-signing
lensWorker.gateway.secretName: existing-lens-signing
lens.enabled: true
lens.service.name: existing-lens
lens.url: http://existing-lens:4318
lens.publicUrl: https://gateway.example.com/lens-ingest
lens.serviceTokenSecret.name: existing-lens-signing
lens.gatewaySecret.name: existing-lens-signing
asserts:
- equal:
path: spec.rules[2].http.paths[0].path

166
helm/litellm/tests/lens_connection_tests.yaml Normal file
View file

@ -0,0 +1,166 @@
suite: lens connection
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
- gateway/configmap.yaml
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: connects the gateway to an external Lens
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: connects the backend to an external Lens
template: backend/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service on the single ingress host
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

86
helm/litellm/tests/lens_endpoint_defaults_tests.yaml
View file

@ -1,86 +0,0 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

208
helm/litellm/tests/lens_modes_tests.yaml
View file

@ -1,208 +0,0 @@
---
suite: Lens deployment modes
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
- ingress.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: keeps the ingestion endpoint on an independently deployed Lens service
template: ingress.yaml
set:
ingress.enabled: true
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: existing-lens
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

54
helm/litellm/tests/lens_modes_validation_tests.yaml
View file

@ -1,54 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: backend/deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

61
helm/litellm/tests/lens_saved_secrets_tests.yaml
View file

@ -1,61 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

196
helm/litellm/tests/lens_service_tests.yaml
View file

@ -1,196 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- gateway/configmap.yaml
- gateway/deployment.yaml
- backend/deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects gateway/deployment.yaml to the shared Lens service
template: gateway/deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: connects backend/deployment.yaml to the shared Lens service
template: backend/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.host: gateway.example
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: gateway/deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: omits Lens connection settings when disabled in backend/deployment.yaml
template: backend/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: gateway/deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference
values:
- ./values/required.yaml

164
helm/litellm/tests/lens_setup_tests.yaml
View file

@ -1,164 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

79
helm/litellm/tests/lens_worker_tests.yaml
View file

@ -1,79 +0,0 @@
suite: Independent Lens release and credentials
templates:
- lens/deployment.yaml
values:
- ./values/required.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
tests:
- it: selects a Lens release independently of the gateway
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
lensWorker.image.tag: 2.3.4
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:2.3.4
- it: retains the pinned Lens chart default when only the gateway changes
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:0.1.0-dev.0
- it: selects development publications explicitly
set:
lensWorker.image.repository: ghcr.io/berriai/lens-dev
lensWorker.image.tag: sha-0123456789abcdef
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
- it: preserves a verified registry mirror and digest
set:
lensWorker.image.repository: registry.example/lens
lensWorker.image.tag: replaced-label
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- it: rejects a malformed digest
set:
lensWorker.image.digest: sha256:invalid
asserts:
- failedTemplate:
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
- it: keeps the legacy enablement switch working
set:
lensWorker.enabled: false
asserts:
- hasDocuments: {count: 0}
- it: keeps isolated local analysis with delegated gateway access
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: LENS_MODE, value: standalone}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- equal:
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
value: true
- equal:
path: spec.template.spec.volumes[0].emptyDir
value: {medium: Memory, sizeLimit: 1Gi}

69
helm/litellm/values.yaml
View file

@ -654,70 +654,15 @@ ui:
topologySpreadConstraints: []
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

1
license_cache.json
View file

File diff suppressed because one or more lines are too long

252
litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py
View file

@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it.
"""
import os
import re
from collections.abc import Callable, Iterator
from collections.abc import Callable
from dataclasses import dataclass
from functools import reduce
from itertools import accumulate
from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME
DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
REFUSAL: Final = "Existing rows would make a pending schema change fail:"
Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
Kind = Literal["too_long"]
@dataclass(frozen=True, slots=True)
@ -54,9 +52,6 @@ class Check:
statement: str
source: str
index: str | None = None
referenced_table: str | None = None
referenced_columns: tuple[str, ...] = ()
expression: str | None = None
@dataclass(frozen=True, slots=True)
@ -100,33 +95,6 @@ class ColumnType:
ColumnTypeLookup = Callable[[str, str], ColumnType | None]
_IDENT: Final = r'"?(\w+)"?'
_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
_CREATE_INDEX: Final = re.compile(
rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
re.IGNORECASE | re.DOTALL,
)
_ALTER_TABLE: Final = re.compile(
rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
)
_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
_ADD_COLUMN: Final = re.compile(
rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
re.IGNORECASE | re.DOTALL,
)
_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
_ADD_FOREIGN_KEY: Final = re.compile(
rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
re.IGNORECASE | re.DOTALL,
)
_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
_COMMENT: Final = re.compile(r"--[^\n]*")
_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
def migration_data_check_timeout_seconds() -> float:
"""Seconds one count query may run before the preflight gives up on verifying it."""
@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float:
return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
def statements(migration_sql: str) -> tuple[str, ...]:
"""The statements of one migration file, comments stripped, empty ones dropped."""
parts: Final = _COMMENT.sub("", migration_sql).split(";")
return tuple(part.strip() for part in parts if part.strip())
def _split_top_level(text: str) -> tuple[str, ...]:
"""Split on the commas outside parentheses."""
depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
cuts: Final = tuple(
position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
)
starts: Final = (0, *(cut + 1 for cut in cuts))
stops: Final = (*cuts, len(text))
pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
return tuple(piece for piece in pieces if piece)
def _plain_columns(column_list: str) -> tuple[str, ...] | None:
"""The column names of an index or constraint column list, None when one entry is an
expression the derivation cannot turn into a count."""
matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
if any(match is None for match in matches):
return None
return tuple(match.group(1) for match in matches if match is not None)
def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
unique, name, table, method, column_list = match.groups()
columns: Final = _plain_columns(column_list)
if columns is None:
return
label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
if method is None or method.lower() == "btree":
yield Check("too_long", table, columns, label, source, index=name)
if unique:
yield Check("duplicates", table, columns, label, source, index=name)
def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
return
if (add_column := _ADD_COLUMN.match(action)) is not None:
column, body = add_column.groups()
if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
yield Check(
"any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
)
return
if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
own: Final = _plain_columns(columns)
theirs: Final = _plain_columns(referenced_columns)
if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
return
yield Check(
"orphans",
table,
own,
_constraint_label(table, constraint, "FOREIGN KEY"),
source,
referenced_table=referenced_table,
referenced_columns=theirs,
)
return
if (unique := _ADD_UNIQUE.match(action)) is not None:
constraint, columns = unique.groups()
unique_columns: Final = _plain_columns(columns)
if unique_columns is not None:
yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
return
if (check := _ADD_CHECK.match(action)) is not None:
constraint, expression, not_valid = check.groups()
if not_valid is None:
yield Check(
"check_violations",
table,
(),
_constraint_label(table, constraint, "CHECK"),
source,
expression=expression,
)
def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
return f'ALTER TABLE "{table}" ADD {named}{kind}'
def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
"""The count queries one SQL statement needs before it can be applied, empty for a
statement that cannot fail on existing rows or that the derivation cannot read."""
if (index := _CREATE_INDEX.match(statement_text)) is not None:
return tuple(_index_checks(index, statement_text, source))
if (alter := _ALTER_TABLE.match(statement_text)) is not None:
table, actions = alter.groups()
return tuple(
check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
) # comprehension-ok: flattening the per-action checks of one statement
return ()
def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
return tuple(
check
for statement_text in statements(migration_sql)
for check in checks_for_statement(statement_text, f"migration {name}")
) # comprehension-ok: flattening the per-statement checks of one migration
def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
"""The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
@ -424,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...],
def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
from psycopg import sql
table: Final = sql.Identifier(schema, check.table)
columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
match check.kind:
case "too_long":
return _entry_size_query(check, schema, types, limit_bytes)
case "duplicates":
not_null: Final = sql.SQL(" AND ").join(
sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
)
return sql.SQL(
"SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
).format(t=table, nn=not_null, cols=columns)
case "nulls":
return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
t=table, col=sql.Identifier(check.columns[0])
)
case "any_row":
return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
case "orphans":
pairs: Final = sql.SQL(" AND ").join(
sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
)
present: Final = sql.SQL(" AND ").join(
sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
)
return sql.SQL(
"SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
case "check_violations":
return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
t=table,
expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file
)
return _entry_size_query(check, schema, types, limit_bytes)
def _applies(
@ -474,15 +296,7 @@ def _applies(
return False
if check.index is not None and _index_is_valid(connection, schema, check.index):
return False
if check.kind == "any_row":
return types is None
if types is None:
return False
if check.kind == "too_long":
return any(column.can_overflow_btree for column in types)
if check.kind == "orphans":
return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
return True
return types is not None and any(column.can_overflow_btree for column in types)
def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
@ -515,21 +329,7 @@ def _run_check(
rows: Final = _int(values[0], 0) if values else 0
if rows == 0:
return None
match check.kind:
case "too_long":
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
case "duplicates":
return BlockedChange(check, rows, "duplicated value groups")
case "nulls":
return BlockedChange(check, rows, "NULL")
case "any_row":
return BlockedChange(
check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
)
case "orphans":
return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
case "check_violations":
return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
def find_blockers(
@ -571,19 +371,7 @@ def find_blockers(
def _remedy(check: Check) -> str:
joined: Final = ", ".join(check.columns)
match check.kind:
case "too_long":
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
case "duplicates":
return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
case "nulls":
return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
case "any_row":
return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
case "orphans":
return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
case "check_violations":
return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
def _describe(finding: Finding) -> str:
@ -607,30 +395,4 @@ def _describe(finding: Finding) -> str:
def report(findings: tuple[Finding, ...]) -> str:
"""The operator-facing refusal: one block per finding, then what to do next."""
lines: Final = "\n".join(_describe(finding) for finding in findings)
return (
f"{REFUSAL}\n{lines}\n"
"No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
)
def pending_change_checks(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
migrations_dir: Path | None,
indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
) -> tuple[Check, ...]:
"""Every check the pending migrations and the request-log indexes need. `migrations_dir`
is None when the schema is pushed rather than migrated, so only the indexes are checked."""
migration_checks: Final = (
tuple(
check
for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
for check in checks_for_migration(name, migration_sql)
) # comprehension-ok: flattening the per-migration checks
if migrations_dir is not None
else ()
)
index_checks: Final = tuple(
check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
) # comprehension-ok: flattening the per-index checks
return migration_checks + index_checks
return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up."

184
litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py Normal file
View file

@ -0,0 +1,184 @@
"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each
inside its own transaction that is always rolled back, so Postgres itself says whether this
database's rows and schema take them. Each migration holds the locks the real upgrade will,
and releases them before the next one runs."""
from __future__ import annotations
import re
import time
from collections.abc import Callable
from dataclasses import dataclass
from functools import reduce
from pathlib import Path
from typing import TYPE_CHECKING, Final
from litellm_proxy_extras.migration_data_checks import pending_migrations
if TYPE_CHECKING:
import psycopg
_CONCURRENTLY: Final = re.compile(
r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE
)
UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"})
Migration = tuple[str, str]
@dataclass(frozen=True, slots=True)
class Applied:
name: str
seconds: float
replayed: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class Rejected:
name: str
error: str
seconds: float
@dataclass(frozen=True, slots=True)
class Baseline:
"""A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying
history, so the dry run asks the same question the boot does. `refusal` is None when the boot
will baseline, otherwise its reason (a custom migration folder, or the schema diff)."""
refusal: str | None
@property
def matches(self) -> bool:
return self.refusal is None
@dataclass(frozen=True, slots=True)
class DryRun:
applied: tuple[Applied, ...]
rejected: Rejected | None
baseline: Baseline | None = None
@property
def ok(self) -> bool:
return self.rejected is None and (self.baseline is None or self.baseline.matches)
def transactional(migration_sql: str) -> str:
"""The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones."""
return _CONCURRENTLY.sub(r"\1", migration_sql)
def _attempt(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
chain: tuple[Migration, ...],
clock: Callable[[], float],
) -> tuple[float, psycopg.Error | None]:
import psycopg
from psycopg import sql
started: Final = clock()
try:
with connection.transaction(force_rollback=True):
connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema)))
for _, migration_sql in chain:
connection.execute(transactional(migration_sql))
except psycopg.Error as exc:
return clock() - started, exc
return clock() - started, None
def _run(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
pending: tuple[Migration, ...],
position: int,
done: DryRun,
clock: Callable[[], float],
) -> DryRun:
"""One migration in its own rolled-back transaction. A migration that fails because something
an earlier pending migration creates does not exist yet is rerun with those replayed in front."""
if done.rejected is not None:
return done
name: Final = pending[position][0]
seconds, error = _attempt(connection, schema, (pending[position],), clock)
if error is None:
return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline)
if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0:
return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline)
earlier: Final = pending[:position]
chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock)
if chained_error is None:
replayed: Final = tuple(earlier_name for earlier_name, _ in earlier)
return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline)
return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline)
def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
from psycopg import sql
row: Final = connection.execute(
"SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),)
).fetchone()
return row is not None and row[0] is not None
def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
row: Final = connection.execute(
"SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1",
(schema,),
).fetchone()
return row is not None
def dry_run(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
migrations_dir: Path,
baseline_check: Callable[[], str | None] | None = None,
clock: Callable[[], float] = time.monotonic,
) -> DryRun:
"""Apply every pending migration in order, each in its own transaction that is rolled back
before the next one starts. The first migration Postgres rejects ends the run with its error.
`baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for
the replay on a database that has tables but no migration history, which the boot baselines."""
if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema):
return DryRun((), None, Baseline(baseline_check()))
pending: Final = pending_migrations(connection, schema, migrations_dir)
return reduce(
lambda done, position: _run(connection, schema, pending, position, done, clock),
range(len(pending)),
DryRun((), None),
)
def _timing(applied: Applied) -> str:
replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else ""
return f"\n {applied.name}: {applied.seconds:.1f} s{replay}"
def report(result: DryRun) -> str:
"""What the operator reads: the per-migration time the real upgrade will hold its locks,
or the migration Postgres rejected with Postgres's own words. Nothing was applied either way."""
if result.baseline is not None:
if result.baseline.matches:
return (
"No migration history and the schema matches this build, "
"the proxy baselines it at boot, nothing to dry-run."
)
return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}"
timings: Final = "".join(_timing(applied) for applied in result.applied)
if result.rejected is None:
count: Final = len(result.applied)
if count == 0:
return "No pending migrations, this database is on this version's schema."
return (
f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next "
f"so its locks are released, nothing changed:{timings}"
)
return (
f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, "
f"nothing was applied.\n{result.rejected.error}"
+ (f"\nApplied before it in the dry run:{timings}" if timings else "")
)

21
litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py
View file

@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
)
def has_packaged_history(migrations_dir: Path) -> bool:
"""Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships,
the only history the boot baselines an existing database onto."""
packaged_dir: Final = Path(__file__).parent
migrations: Final = migration_files(migrations_dir)
return (
bool(migrations)
and migrations == migration_files(packaged_dir)
and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes()
)
def baseline_current_schema(
coordinator: MigrationCoordinator,
schema: str,
@ -95,14 +107,9 @@ def baseline_current_schema(
) -> None:
from psycopg import sql
packaged_dir: Final = Path(__file__).parent
migrations: Final = migration_files(migrations_dir)
if (
not migrations
or migrations != migration_files(packaged_dir)
or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
):
if not has_packaged_history(migrations_dir):
raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
migrations: Final = migration_files(migrations_dir)
coordinator.acquire_prisma_lock()
existing: Final = coordinator.connection.execute(

21
litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql Normal file
View file

@ -0,0 +1,21 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
"subscription_account_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"account_id" TEXT NOT NULL,
"label" TEXT,
"monthly_fee" DOUBLE PRECISION NOT NULL,
"currency" TEXT NOT NULL,
"billing_period_start" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

16
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

107
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -11,18 +11,13 @@ from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
from typing import TYPE_CHECKING, Final, Protocol
from litellm_proxy_extras import prisma_toolchain
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras._redaction import _redact_credentials
from litellm_proxy_extras.migration_data_checks import (
Finding,
find_blockers,
migration_data_check_timeout_seconds,
pending_change_checks,
report,
)
from litellm_proxy_extras.migration_dry_run import dry_run
from litellm_proxy_extras.migration_dry_run import report as dry_run_report
from litellm_proxy_extras.migration_lock import held_migration_lock
from litellm_proxy_extras.prisma_toolchain import (
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import (
apply_replica_identity_full,
)
from litellm_proxy_extras.request_log_indexes import (
REQUEST_LOG_INDEXES,
ensure_request_log_indexes,
filter_request_log_index_diff,
)
@ -52,7 +46,7 @@ class LensCheckConnect(Protocol):
) -> "psycopg.Connection[tuple[object, ...]]": ...
def str_to_bool(value: Optional[str]) -> bool:
def str_to_bool(value: str | None) -> bool:
if value is None:
return False
return value.lower() in ("true", "1", "t", "y", "yes")
@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str:
).strip()
def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
def _without_spend_logs_pk_clauses(statement: str) -> str | None:
prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
if not prefix_match:
return statement
@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int:
return max(_migration_timestamp(n) for n in names)
def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]:
if isinstance(command, tuple):
return tuple(_redact_credentials(str(argument)) for argument in command)
if isinstance(command, list):
@ -404,7 +398,7 @@ class ProxyExtrasDBManager:
return tuple(row) if row is not None else ()
@staticmethod
def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
def _failed_migration_logs(migration_name: str, started_at: str) -> str | None:
row: Final = ProxyExtrasDBManager._read_migration_ledger(
"SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
"AND finished_at IS NULL AND rolled_back_at IS NULL",
@ -721,10 +715,10 @@ class ProxyExtrasDBManager:
return row is not None
@staticmethod
def _prisma_schema_param(url: str) -> Optional[str]:
def _prisma_schema_param(url: str) -> str | None:
"""The `schema` query param Prisma uses to pick its target schema,
or None when the URL does not set one."""
from urllib.parse import urlparse, parse_qsl
from urllib.parse import parse_qsl, urlparse
return next(
(v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager:
bool: True if setup was successful, False otherwise
"""
ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
if not migrated:
return False
@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager:
return True
@staticmethod
def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
"""What in the existing rows would make a pending migration or request-log index
fail, read with count queries only: no write, no table lock. Empty when nothing
blocks, when no database is configured, or when it cannot be reached, so the
migration step that follows reports the connection problem the way it does today."""
def _baseline_refusal(prisma_dir: Path) -> str | None:
"""Why `baseline_current_schema` would refuse this database at boot, None when it would baseline:
the migrations folder must be the packaged one, then `prisma migrate diff` must find the live
database and this build's schema.prisma identical."""
from litellm_proxy_extras.migration_lock import migration_environment
from litellm_proxy_extras.migration_recovery import has_packaged_history
if not has_packaged_history(prisma_dir):
return (
f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the "
"migrations this build ships, and the proxy only baselines onto the packaged history."
)
try:
prisma_toolchain.run_prisma(
(
_get_prisma_command(),
"migrate",
"diff",
"--from-schema-datasource",
str(prisma_dir / "schema.prisma"),
"--to-schema-datamodel",
str(prisma_dir / "schema.prisma"),
"--exit-code",
),
timeout=prisma_toolchain.prisma_command_timeout(),
env=migration_environment(_get_prisma_env()),
)
except subprocess.CalledProcessError as exc:
diff: Final = str(exc.stdout or exc.stderr or exc).strip()
return f"the schema differs from this build. prisma migrate diff:\n{diff}"
except subprocess.TimeoutExpired as exc:
return f"prisma migrate diff did not finish within {exc.timeout} s"
return None
@staticmethod
def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
"""`--check_migrations_against_existing_data`: the pending migrations applied for real on
DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports
what the upgrade would hit. (True, the per-migration timings) when they all apply,
(False, Postgres's own error) when one does not."""
import psycopg
if not use_migrate:
return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.")
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return ()
return (False, "DATABASE_URL is not set.")
direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
os.environ.get("DIRECT_URL") or database_url
)
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir())
try:
with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
result: Final = dry_run(
connection,
schema,
prisma_dir / "migrations",
baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir),
)
except psycopg.OperationalError as exc:
logger.warning(
"Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
)
return ()
@staticmethod
def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
"""The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
return report(findings) if findings else None
@staticmethod
def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
"""Refuse to run migrations that the existing rows would make fail, before any is
applied. The RuntimeError carries the report and reaches the operator through the
proxy's "Database migration cannot proceed" exit."""
refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
if refusal is not None:
raise RuntimeError(refusal)
return (False, f"Could not reach the database: {_redact_credentials(str(exc))}")
return (result.ok, dry_run_report(result))
@staticmethod
def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager:
time.sleep(random.randrange(5, 15))
finally:
os.chdir(original_dir)
pass
return False

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.111"
version = "0.4.112"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.111"
version = "0.4.112"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

1
litellm-rust/Cargo.lock generated
View file

@ -4345,6 +4345,7 @@ dependencies = [
"litellm-inference-transcription",
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
"litellm-router-types",
"litellm-secrets",
"litellm-secrets-aws",

18
litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs
View file

@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[
|mapping| {
mapping.error_str.contains("invalid_request_error")
&& !mapping.error_str.contains("Incorrect API key provided")
&& !matches!(mapping.status, Some(401 | 404))
},
PublicError::BadRequest,
),
@ -132,6 +133,23 @@ mod tests {
assert_eq!(classified(Some(400), text), Some(expected));
}
#[rstest::rstest]
#[case::unauthorized(Some(401), None)]
#[case::not_found(Some(404), None)]
#[case::forbidden(Some(403), Some(PublicError::BadRequest))]
#[case::rate_limited(Some(429), Some(PublicError::BadRequest))]
#[case::server_error(Some(500), Some(PublicError::BadRequest))]
#[case::no_status(None, Some(PublicError::BadRequest))]
fn invalid_request_errors_leave_401_and_404_to_the_status_mapping(
#[case] status: Option<u16>,
#[case] expected: Option<PublicError>,
) {
assert_eq!(
classified(status, "invalid_request_error bad field"),
expected
);
}
#[rstest::rstest]
#[case::rate_limit_before_context_window(
"rate limit and This model's maximum context length is 10",

133
litellm-rust/crates/inference-ocr/tests/ocr/azure_document_intelligence.rs
View file

@ -12,6 +12,7 @@ use rstest::rstest;
use super::*;
const MODEL: &str = "azure_ai/doc-intelligence/prebuilt-read";
const LAYOUT_MODEL: &str = "azure_ai/doc-intelligence/prebuilt-layout";
fn read_request(base: &str, options: Value) -> LiteLLMOcrRequest {
ocr_request(MODEL, base, options)
@ -70,6 +71,7 @@ async fn pages_features_and_extra_options_map_to_the_analyze_call() {
#[case(json!({"pages": "1&&features=bad"}), Error::Pages(PagesFault::Range))]
#[case(json!({"features": "languages&pages=1"}), Error::Features)]
#[case(json!({"req_format": "azure"}), Error::RequestFormat)]
#[case(json!({"outputContentFormat": "html"}), Error::OutputContentFormat)]
#[tokio::test]
async fn invalid_pages_features_and_format_are_rejected_before_sending(
#[case] options: Value,
@ -130,6 +132,137 @@ async fn an_inline_document_is_sent_as_base64_and_only_page_text_is_kept(#[case]
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
}
#[rstest]
#[case::lower_case("markdown")]
#[case::capitalized("Markdown")]
#[tokio::test]
async fn markdown_output_is_requested_on_the_analyze_call_and_split_into_pages(
#[case] output_content_format: &str,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {
"content": "# Tïtle\n\n<!-- PageBreak -->\n\n| a |",
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [
{"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]},
{"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]}
]
}
}))])
.await;
let response = perform(ocr_request(
LAYOUT_MODEL,
&upstream.uri(),
json!({"outputContentFormat": output_content_format, "pages": [0, 1]}),
))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
Some("markdown")
);
assert_eq!(
sent.query("stringIndexType").as_deref(),
Some("unicodeCodePoint")
);
assert_eq!(sent.query("pages").as_deref(), Some("1,2"));
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(
response.content.as_deref(),
Some("# Tïtle\n\n<!-- PageBreak -->\n\n| a |")
);
let pages = response
.pages
.iter()
.map(|page| page.markdown.as_str())
.collect::<Vec<_>>();
assert_eq!(pages, ["# Tïtle\n\n<!-- PageBreak -->\n\n", "| a |"]);
assert_eq!(
response.extra_fields.get("stringIndexType"),
Some(&json!("unicodeCodePoint"))
);
assert_eq!(
response.usage_info.as_ref().unwrap().pages_processed,
Some(2)
);
}
#[rstest]
#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))]
#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))]
#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))]
#[case::omitted(LAYOUT_MODEL, json!({}), None)]
#[tokio::test]
async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets(
#[case] model: &str,
#[case] options: Value,
#[case] output_content_format: Option<&str>,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {
"contentFormat": "text",
"stringIndexType": "textElements",
"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]
}
}))])
.await;
let response = perform(ocr_request(model, &upstream.uri(), options))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
output_content_format
);
assert_eq!(sent.query("stringIndexType"), None);
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(response.pages[0].markdown, "hello");
assert_eq!(response.extra_fields.get("stringIndexType"), None);
}
#[rstest]
#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))]
#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))]
#[case::empty(json!({"outputContentFormat": ""}), None)]
#[case::empty_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
Some("markdown")
)]
#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)]
#[tokio::test]
async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling(
#[case] options: Value,
#[case] output_content_format: Option<&str>,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]}
}))])
.await;
let response = perform(ocr_request(MODEL, &upstream.uri(), options))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
output_content_format
);
assert_eq!(sent.query("output_content_format"), None);
assert_eq!(sent.query("drop_params"), None);
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(response.pages[0].markdown, "hello");
}
#[tokio::test]
async fn native_format_normalizes_pages_and_keeps_the_provider_response() {
let operation = json!({

23
litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs
View file

@ -1,14 +1,31 @@
use serde_json::Value;
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionUsage {
pub input_tokens: u64,
pub output_tokens: u64,
pub total_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
pub usage: Option<AudioTranscriptionUsage>,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
match self.usage {
None => serde_json::json!({"text": self.text}),
Some(usage) => serde_json::json!({
"text": self.text,
"usage": {
"type": "tokens",
"input_tokens": usage.input_tokens,
"output_tokens": usage.output_tokens,
"total_tokens": usage.total_tokens,
},
}),
}
}
}

4
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -238,7 +238,8 @@ impl AnthropicBeta {
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
| Self::ThinkingBindingControls20260801
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -275,7 +276,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,

426
litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs
View file

@ -1,4 +1,7 @@
use std::{collections::BTreeSet, time::Duration};
use std::{
collections::{BTreeMap, BTreeSet},
time::Duration,
};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_auth::{InputSource, Sourced};
@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
use serde_with::{DefaultOnError, serde_as};
use tokio::time::Instant;
use crate::base_llm::ocr::{
@ -17,9 +20,10 @@ use crate::base_llm::ocr::{
handler::{CallHooks, OcrClient, read_json_response},
settings::OcrSettings,
transformation::{
BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS,
OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest,
ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES,
OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext,
PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
drops_unsupported_params, response_format,
},
};
use litellm_llms_types::formats::ocr::{
@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";
const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT";
const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat";
const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format";
const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint";
const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType";
const MARKDOWN_MODEL_ID: &str = "prebuilt-layout";
#[derive(
Clone,
Copy,
Debug,
PartialEq,
strum::EnumString,
strum::IntoStaticStr,
strum::Display,
strum::VariantArray,
serde_with::SerializeDisplay,
)]
#[strum(ascii_case_insensitive)]
pub enum OutputContentFormat {
#[strum(serialize = "text")]
Text,
#[strum(serialize = "markdown")]
Markdown,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct DocumentIntelligenceParams {
@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams {
pub pages: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub features: Option<String>,
#[serde(
rename = "outputContentFormat",
skip_serializing_if = "Option::is_none"
)]
pub output_content_format: Option<OutputContentFormat>,
#[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")]
pub string_index_type: Option<&'static str>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation {
#[derive(Clone, Debug, Default, Deserialize)]
struct AzureDocumentIntelligenceAnalyzeResult {
pub content: Option<String>,
#[serde(rename = "contentFormat")]
pub content_format: Option<String>,
#[serde(rename = "stringIndexType")]
pub string_index_type: Option<String>,
#[serde(default)]
pub pages: Vec<AzureDocumentIntelligencePage>,
pub tables: Option<Vec<Map<String, Value>>>,
@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage {
pub unit: Option<String>,
#[serde(default)]
pub lines: Vec<AzureDocumentIntelligenceLine>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub spans: Option<Vec<AzureDocumentIntelligenceSpan>>,
}
#[derive(Clone, Debug, Deserialize)]
@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine {
pub content: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
struct AzureDocumentIntelligenceSpan {
pub offset: usize,
pub length: usize,
}
impl AzureDocumentIntelligenceSpan {
fn end(&self) -> usize {
self.offset.saturating_add(self.length)
}
}
#[derive(Clone, Debug)]
pub struct AzureDocumentIntelligenceOcrConfig;
@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
type Environment = Vec<(String, String)>;
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
&["pages", "features", "req_format"]
&[
"pages",
"features",
OUTPUT_CONTENT_FORMAT_PARAM,
OUTPUT_CONTENT_FORMAT_ALIAS,
"req_format",
DROP_PARAMS_PARAM,
]
}
fn get_api_key_env_var(&self) -> Option<&'static str> {
@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
fn map_ocr_params(
&self,
non_default_params: &CallArguments,
_model: &str,
model: &str,
) -> Result<DocumentIntelligenceParams, Error> {
let output_content_format = requested_output_content_format(non_default_params)?;
let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown)
&& response_format(non_default_params)? != OcrResponseFormat::Native
&& model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID);
Ok(DocumentIntelligenceParams {
pages: normalize_pages_param(non_default_params.get("pages"))?,
features: normalize_features_param(non_default_params.get("features"))?,
output_content_format,
string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX),
})
}
@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result<Option<String>,
Ok(Some(normalized.join(",")))
}
fn requested_output_content_format(
params: &CallArguments,
) -> Result<Option<OutputContentFormat>, Error> {
let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS]
.into_iter()
.filter_map(|name| params.get(name))
.find(|format| !is_unset(format));
match requested.map(parse_output_content_format).transpose() {
Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None),
parsed => parsed,
}
}
fn is_unset(format: &Value) -> bool {
match format {
Value::Null => true,
Value::String(format) => format.trim().is_empty(),
_ => false,
}
}
fn parse_output_content_format(format: &Value) -> Result<OutputContentFormat, Error> {
format
.as_str()
.ok_or(Error::OutputContentFormat)?
.trim()
.parse()
.map_err(|_| Error::OutputContentFormat)
}
fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, Error> {
let source = document.source();
if source.is_empty() {
@ -367,10 +464,27 @@ fn transform_completed_response(
));
}
let result = response.analyze_result.unwrap_or_default();
let markdown_content = result
.content
.as_deref()
.filter(|_| {
result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown))
&& result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX)
&& result.pages.iter().all(|page| page.spans.is_some())
})
.map(|content| {
CodePointText::new(
content,
result
.pages
.iter()
.flat_map(|page| page.spans.iter().flatten()),
)
});
let pages = result
.pages
.into_iter()
.map(|page| transform_azure_page(page, dpi))
.map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi))
.collect::<Result<Vec<_>, _>>()?;
let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
Ok(LiteLLMOcrResponse {
@ -381,11 +495,20 @@ fn transform_completed_response(
pages_processed: Some(pages_processed),
..Default::default()
}),
extra_fields: result
.string_index_type
.filter(|unit| unit == CODE_POINT_STRING_INDEX)
.map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))]))
.unwrap_or_default(),
..LiteLLMOcrResponse::new(model, pages)
})
}
fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
fn transform_azure_page(
page: AzureDocumentIntelligencePage,
markdown_content: Option<&CodePointText<'_>>,
dpi: i64,
) -> Result<OcrPage, Error> {
let index = page
.page_number
.unwrap_or(1)
@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
page.unit.as_deref().unwrap_or("inch"),
dpi,
)?;
let markdown = page
.lines
.iter()
.map(|line| line.content.as_deref().unwrap_or_default())
.collect::<Vec<_>>()
.join("\n");
let markdown = match markdown_content {
Some(content) => page
.spans
.iter()
.flatten()
.map(|span| content.slice(span))
.collect::<String>(),
None => page
.lines
.iter()
.map(|line| line.content.as_deref().unwrap_or_default())
.collect::<Vec<_>>()
.join("\n"),
};
Ok(OcrPage {
index,
markdown,
@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
})
}
struct CodePointText<'a> {
text: &'a str,
byte_offsets: BTreeMap<usize, usize>,
}
impl<'a> CodePointText<'a> {
fn new<'s>(
text: &'a str,
spans: impl Iterator<Item = &'s AzureDocumentIntelligenceSpan>,
) -> Self {
let span_bounds = spans
.flat_map(|span| [span.offset, span.end()])
.collect::<BTreeSet<_>>();
Self {
text,
byte_offsets: text
.char_indices()
.map(|(byte_offset, _)| byte_offset)
.enumerate()
.filter(|(code_point, _)| span_bounds.contains(code_point))
.collect(),
}
}
fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str {
let byte_offset = |code_point: usize| {
self.byte_offsets
.get(&code_point)
.copied()
.unwrap_or(self.text.len())
};
self.text
.get(byte_offset(span.offset)..byte_offset(span.end()))
.unwrap_or_default()
}
}
fn convert_dimensions(
width: f64,
height: f64,
@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig {
.features
.iter()
.map(|features| ("features", features.as_str())),
)
.chain(
params
.output_content_format
.map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())),
)
.chain(
params
.string_index_type
.map(|index_type| ("stringIndexType", index_type)),
),
)
.into_string()
@ -697,8 +875,12 @@ mod tests {
}
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout")
}
fn map_for_model(value: Value, model: &str) -> Result<DocumentIntelligenceParams, Error> {
let arguments = serde_json::from_value(value).unwrap();
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model)
}
#[test]
@ -828,6 +1010,218 @@ mod tests {
assert_eq!(map(json!({"features": []})).unwrap().features, None);
}
#[rstest]
#[case::markdown(
json!({"outputContentFormat": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::markdown_litellm_format(
json!({"outputContentFormat": "markdown", "req_format": "litellm"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::markdown_native_format(
json!({"outputContentFormat": "markdown", "req_format": "native"}),
Some(OutputContentFormat::Markdown),
None
)]
#[case::capitalized_markdown(
json!({"outputContentFormat": "Markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)]
#[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)]
#[case::padded_markdown(
json!({"outputContentFormat": " markdown\n"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::snake_case_markdown(
json!({"output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::camel_case_wins_over_snake_case(
json!({"outputContentFormat": "text", "output_content_format": "markdown"}),
Some(OutputContentFormat::Text),
None
)]
#[case::empty_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::null_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": null, "output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::empty(json!({"outputContentFormat": ""}), None, None)]
#[case::blank(json!({"outputContentFormat": " "}), None, None)]
#[case::null(json!({"outputContentFormat": null}), None, None)]
#[case::omitted(json!({}), None, None)]
fn output_content_format_maps_to_query_fields(
#[case] options: Value,
#[case] format: Option<OutputContentFormat>,
#[case] string_index_type: Option<&str>,
) {
let mapped = map(options).unwrap();
assert_eq!(mapped.output_content_format, format);
assert_eq!(mapped.string_index_type, string_index_type);
}
#[rstest]
#[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))]
#[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))]
#[case::read("azure_ai/doc-intelligence/prebuilt-read", None)]
#[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)]
#[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)]
fn only_the_layout_model_asks_for_code_point_offsets(
#[case] model: &str,
#[case] string_index_type: Option<&str>,
) {
let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap();
assert_eq!(
mapped.output_content_format,
Some(OutputContentFormat::Markdown)
);
assert_eq!(mapped.string_index_type, string_index_type);
}
#[rstest]
#[case::unknown(json!({"outputContentFormat": "html"}))]
#[case::number(json!({"outputContentFormat": 1}))]
#[case::boolean(json!({"outputContentFormat": true}))]
#[case::list(json!({"outputContentFormat": ["markdown"]}))]
#[case::snake_case_unknown(json!({"output_content_format": "html"}))]
#[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))]
fn invalid_output_content_format_is_a_request_error(#[case] options: Value) {
let error = map(options).unwrap_err();
assert!(matches!(error, Error::OutputContentFormat), "{error:?}");
assert_eq!(error.http_status_code(), Some(400));
}
#[rstest]
#[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))]
#[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))]
#[case::snake_case_unknown(
json!({"output_content_format": "html", "drop_params": true, "pages": [0]})
)]
fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) {
let mapped = map(options).unwrap();
assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"}));
}
#[rstest]
#[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))]
#[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))]
fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) {
let mapped = map(options).unwrap();
assert_eq!(
mapped.output_content_format,
Some(OutputContentFormat::Markdown)
);
}
fn markdown_pages(analyze_result: Value) -> Vec<String> {
let raw =
serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result}))
.unwrap();
AzureDocumentIntelligenceOcrConfig
.transform_ocr_response("model", &raw, OcrResponseFormat::Litellm)
.unwrap()
.pages
.into_iter()
.map(|page| page.markdown)
.collect()
}
#[rstest]
fn markdown_pages_are_code_point_slices_of_the_content() {
let content = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n## Två 🙂\n\n|a|\n\n> tail";
let second_page_start = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n".chars().count();
let pages = markdown_pages(json!({
"content": content,
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [
{"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}],
"lines": [{"content": "Ünïcode क्ष"}]},
{"pageNumber": 2, "spans": [
{"offset": second_page_start, "length": 8},
{"offset": second_page_start + 10, "length": 3}
], "lines": [{"content": "Två"}]}
]
}));
assert_eq!(
pages,
["# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n", "## Två 🙂|a|"]
);
}
#[rstest]
fn spans_past_the_content_end_are_clamped() {
let pages = markdown_pages(json!({
"content": "ab",
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}]
}));
assert_eq!(pages, ["b"]);
}
#[rstest]
#[case::text_content(json!("text"), json!("unicodeCodePoint"))]
#[case::grapheme_offsets(json!("markdown"), json!("textElements"))]
#[case::no_index_type(json!("markdown"), Value::Null)]
#[case::no_content_format(Value::Null, json!("unicodeCodePoint"))]
fn other_responses_keep_line_joined_pages(
#[case] content_format: Value,
#[case] string_index_type: Value,
) {
let pages = markdown_pages(json!({
"content": "# A\n\nB",
"contentFormat": content_format,
"stringIndexType": string_index_type,
"pages": [{"spans": [{"offset": 0, "length": 6}],
"lines": [{"content": "A"}, {"content": "B"}]}]
}));
assert_eq!(pages, ["A\nB"]);
}
#[rstest]
#[case::null_spans(json!("markdown"), json!(null))]
#[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))]
#[case::missing_length(json!("markdown"), json!([{"offset": 0}]))]
#[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))]
#[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))]
fn unusable_page_spans_keep_line_joined_pages(
#[case] content_format: Value,
#[case] broken_spans: Value,
) {
let pages = markdown_pages(json!({
"content": "# A\n\nB",
"contentFormat": content_format,
"stringIndexType": "unicodeCodePoint",
"pages": [
{"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]},
{"spans": broken_spans, "lines": [{"content": "B"}]}
]
}));
assert_eq!(pages, ["A", "B"]);
}
#[tokio::test]
async fn request_endpoint_cannot_receive_environment_key() {
let connection = OcrConnection {

8
litellm-rust/crates/llms/src/base_llm/ocr/error.rs
View file

@ -59,6 +59,10 @@ pub enum Error {
Pages(PagesFault),
#[error("invalid OCR features")]
Features,
#[error(
"invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`."
)]
OutputContentFormat,
#[error("OCR model cannot be a dot segment")]
DotModel,
#[error("OCR response exceeds the size limit of {limit} bytes")]
@ -200,6 +204,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::TooLarge { .. }
| Self::ResponseField { .. }
@ -242,6 +247,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::TooLarge { .. }
| Self::ResponseField { .. }
@ -291,6 +297,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::InvalidProvider(_)
| Self::InvalidModel { .. }
@ -345,6 +352,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::InvalidProvider(_)
| Self::InvalidModel { .. }

5
litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs
View file

@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
pub const OCR_POLL_RETRY_SECS: u64 = 2;
pub const DROP_PARAMS_PARAM: &str = "drop_params";
#[derive(Clone, Default)]
pub struct OcrCredentialInputs {
@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result<OcrResponseFor
.map(|format| format.unwrap_or_default())
}
pub fn drops_unsupported_params(optional_params: &CallArguments) -> bool {
optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true))
}
#[derive(Debug)]
pub struct DecodedOcrResponse<T> {
pub data: T,

83
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -6,8 +6,8 @@ use litellm_auth_aws::{
};
use litellm_core_utils::core_helpers::json_type_name;
use litellm_llms_types::{
formats::audio_transcription::AudioTranscriptionResponseData,
providers::bedrock::ConverseResponse,
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
providers::bedrock::{ConverseResponse, ConverseUsage},
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
if let Some(prompt) = optional_string(&optional_params, "prompt") {
instruction.push_str(&format!(" Additional context: {prompt}"));
}
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
if let Some(temperature) = optional_params.get("temperature") {
inference_config.insert("temperature".to_string(), temperature.clone());
}
let temperature = optional_params
.get("temperature")
.cloned()
.unwrap_or(json!(0));
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
}),
})
}
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
_model: &str,
response_json: Value,
) -> Result<AudioTranscriptionResponseData, Error> {
let reports_usage = response_json.get("usage").is_some();
let response: ConverseResponse =
serde_json::from_value(response_json).map_err(|error| {
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
Ok(AudioTranscriptionResponseData {
text: response.content_text(),
usage: reports_usage.then(|| transcription_usage(&response.usage)),
})
}
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
}
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
AudioTranscriptionUsage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
total_tokens: usage
.total_tokens
.unwrap_or(usage.input_tokens + usage.output_tokens),
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
fn no_env(_: &str) -> Option<String> {
None
}
#[test]
fn request_matches_python_shape() {
let params = Map::from_iter([
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("temperature".to_string(), json!(0)),
("timestamp_granularities".to_string(), json!(["word"])),
]);
#[rstest]
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
#[case::default_temperature(None, json!(0))]
fn request_sends_audio_and_instruction_without_system_block(
#[case] temperature: Option<Value>,
#[case] expected_temperature: Value,
) {
let params = Map::from_iter(
[
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("timestamp_granularities".to_string(), json!(["word"])),
]
.into_iter()
.chain(temperature.map(|value| ("temperature".to_string(), value))),
);
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_request(
@ -225,22 +245,39 @@ mod tests {
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
})
);
}
#[test]
fn response_concatenates_content_blocks() {
#[rstest]
#[case::reported_total(
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
)]
#[case::derived_total(
json!({"inputTokens": 1, "outputTokens": 2}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
)]
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
)
.expect("response");
assert_eq!(result.text, "hello world");
assert_eq!(result.into_json(), json!({"text": "hello world"}));
assert_eq!(result.into_json(), expected);
}
#[rstest]
fn response_without_usage_reports_none() {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
)
.expect("response");
assert_eq!(result.into_json(), json!({"text": "hello"}));
}
#[test]

2
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
## Tests
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`

2
litellm-rust/crates/model-catalog/Cargo.toml
View file

@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
[dependencies]
litellm-cost = { workspace = true, features = ["serde"] }
litellm-error.workspace = true
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
@ -24,5 +25,4 @@ time.workspace = true
[dev-dependencies]
jsonschema = { version = "0.55.1", default-features = false }
litellm-error.workspace = true
rstest.workspace = true

26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs Normal file
View file

@ -0,0 +1,26 @@
use std::{env, fs, path::PathBuf, process::ExitCode};
use litellm_model_catalog::Catalog;
fn main() -> ExitCode {
let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
eprintln!("usage: check-registry <model_prices_and_context_window.json>");
return ExitCode::from(2);
};
let body = match fs::read(&path) {
Ok(body) => body,
Err(error) => {
eprintln!("{}: {error}", path.display());
return ExitCode::FAILURE;
}
};
let problems = Catalog::problems(&body);
for problem in &problems {
eprintln!("{}", litellm_error::Report::new(problem));
}
if problems.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
}
}

114
litellm-rust/crates/model-catalog/src/catalog.rs
View file

@ -5,7 +5,7 @@ use indexmap::IndexMap;
use serde::Deserialize;
use serde_json::Value;
use crate::error::{Error, Loaded, Rejects, Repr};
use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
use crate::fallback::{FallbackGeneralizations, FallbackRule};
use crate::model::Model;
use crate::store::{Rows, Store};
@ -211,6 +211,36 @@ impl Catalog {
})
}
pub fn problems(body: &[u8]) -> Vec<Error> {
let Loaded {
value: catalog,
errors,
} = match Self::parse(body) {
Ok(loaded) => loaded,
Err(fatal) => return vec![fatal],
};
let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
EntryWire::read(entry.fields())
.unknown
.into_iter()
.map(move |(key, _)| {
Error::new(
pointer_child(&pointer_for_key(model), &key),
Repr::UnreadField,
)
})
});
let no_fallback_rules = catalog
.fallback_rules()
.is_none_or(<[_]>::is_empty)
.then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
errors
.into_iter()
.chain(unread)
.chain(no_fallback_rules)
.collect()
}
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
let (_, _, errors) = read_row(fields);
errors
@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
)
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
#[rstest]
#[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
#[case::not_json("{".to_owned(), vec![("", "json")])]
#[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
#[case::unread_key(
format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
vec![("/m/typo_cost", "unread_field")]
)]
#[case::empty_model_key(
format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
vec![("//future_field", "unread_field")]
)]
#[case::unread_key_on_invalid_row(
format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
)]
#[case::escaped_unread_key_on_invalid_row(
format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
)]
#[case::no_rules(
r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
#[case::empty_rules(
r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
fn problems_list_what_a_cost_map_change_must_fix(
#[case] body: String,
#[case] expected: Vec<(&str, &str)>,
) {
let problems: Vec<_> = Catalog::problems(body.as_bytes())
.iter()
.map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
.collect();
let expected: Vec<_> = expected
.into_iter()
.map(|(path, kind)| (path.to_owned(), kind))
.collect();
assert_eq!(problems, expected);
}
#[rstest]
fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
let body = br#"{
"good":{"litellm_provider":"test","custom_metadata":true},
"bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
}"#;
let loaded = Catalog::parse(body).unwrap();
assert_eq!(loaded.errors.len(), 1);
assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
assert_eq!(rejected.len(), 1);
assert!(catalog.lookup("bad").is_none());
let good = catalog.lookup("good").unwrap();
assert!(good.entry.model().is_some());
assert_eq!(
good.entry.field("custom_metadata"),
Some(&Value::Bool(true))
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
);
}
}

10
litellm-rust/crates/model-catalog/src/error.rs
View file

@ -109,6 +109,12 @@ pub(crate) enum Repr {
#[source]
cause: Option<DateCause>,
},
#[error("not a field litellm reads")]
#[strum_discriminants(strum(serialize = "unread_field"))]
UnreadField,
#[error("has no rules")]
#[strum_discriminants(strum(serialize = "no_fallback_rules"))]
NoFallbackRules,
}
/// What a problem takes out of the served map
@ -148,7 +154,7 @@ impl Error {
| Repr::NoOffPeakHours
| Repr::EmptyList
| Repr::InvalidDate { .. } => Rejects::Row,
Repr::MissingProvider => Rejects::Nothing,
Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
}
}
}
@ -334,6 +340,8 @@ mod tests {
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
#[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
#[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
assert_eq!(Error::from(error).rejects(), expected);
}

Some files were not shown because too many files have changed in this diff Show more