mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
chore: merge main into the hosted_vllm batch fix
Main's batch usage validator and serializer annotate with object, which the object field on LiteLLMBatch shadows inside the class, so they now use the shared JSON value alias like the rest of the class
This commit is contained in:
commit
1ccff0f9ae
594 changed files with 28400 additions and 8651 deletions
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GITHUB_ACTIONS:-}" = true ]; then
|
||||
echo "Integration contracts are owned by CircleCI" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
suite="${1:?integration suite required}"
|
||||
mode="${2:-standard}"
|
||||
side="${3:-}"
|
||||
requested_workers="${INTEGRATION_WORKERS:-}"
|
||||
license_env=()
|
||||
if [ -n "${LITELLM_LICENSE:-}" ]; then
|
||||
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
|
||||
fi
|
||||
if [ "$mode" = replica ]; then
|
||||
results="test-results/integration-${suite}-replica"
|
||||
elif [ "$mode" = parity ]; then
|
||||
|
|
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
|
|||
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
|
||||
test -f "$LITELLM_UI_PATH/index.html"
|
||||
fi
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
|
||||
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
|
||||
|
|
@ -107,7 +107,7 @@ sudo iptables -N integration_only
|
|||
guard_created=true
|
||||
sudo iptables -A integration_only -o lo -j ACCEPT
|
||||
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
for service in postgres-db redis-cache; do
|
||||
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
|
||||
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
|
||||
sudo iptables -A integration_only -d "$address" -j ACCEPT
|
||||
done
|
||||
|
|
@ -145,6 +145,9 @@ fi
|
|||
if [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
|
||||
fi
|
||||
if [ -n "$requested_workers" ]; then
|
||||
export INTEGRATION_WORKERS="$requested_workers"
|
||||
fi
|
||||
isolated_workers=false
|
||||
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
|
||||
isolated_workers=true
|
||||
|
|
@ -185,7 +188,7 @@ start_proxy() {
|
|||
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
|
||||
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
|
||||
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
|
||||
|
|
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|||
"${shared_services[@]}" "${worker_coverage[@]}" \
|
||||
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
||||
LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
||||
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
|
||||
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
|
||||
|
|
|
|||
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TextIO
|
||||
|
||||
SECRET_VARIABLE: Final = "SCANNED_SECRET"
|
||||
MIN_ENCODED_LENGTH: Final = 12
|
||||
MAX_ARCHIVE_DEPTH: Final = 3
|
||||
MASK: Final = "***"
|
||||
|
||||
Forms = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hit:
|
||||
location: str
|
||||
form: str
|
||||
|
||||
|
||||
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
|
||||
for offset in range(3):
|
||||
chunk: Final = raw[offset:]
|
||||
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
|
||||
|
||||
|
||||
def secret_forms(secret: str) -> Forms:
|
||||
encoded: Final = (
|
||||
("url-encoded", urllib.parse.quote(secret, safe="")),
|
||||
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
|
||||
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
|
||||
)
|
||||
by_value: Final = MappingProxyType(
|
||||
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
|
||||
)
|
||||
return (("raw", secret), *((name, value) for value, name in by_value.items()))
|
||||
|
||||
|
||||
def mask(text: str, forms: Forms) -> str:
|
||||
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
|
||||
|
||||
|
||||
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value in name:
|
||||
yield Hit(mask(location, forms), f"{form} in name")
|
||||
|
||||
|
||||
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value.encode() in data:
|
||||
yield Hit(mask(location, forms), form)
|
||||
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
|
||||
return
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
for member in archive.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
|
||||
|
||||
|
||||
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
|
||||
for directory, subdirectories, files in os.walk(root):
|
||||
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
|
||||
yield from ((Path(directory, name), False) for name in files)
|
||||
|
||||
|
||||
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
|
||||
location: Final = str(path.relative_to(root))
|
||||
if linked_directory:
|
||||
yield Hit(mask(location, forms), "directory symlink, not scanned")
|
||||
return
|
||||
yield from _name_hits(location, location, forms)
|
||||
yield from _content_hits(location, path.read_bytes(), forms, 0)
|
||||
|
||||
|
||||
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
|
||||
forms: Final = secret_forms(secret)
|
||||
entries: Final = tuple(_walk(root))
|
||||
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
|
||||
return len(entries), hits
|
||||
|
||||
|
||||
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
|
||||
if len(argv) != 1:
|
||||
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
|
||||
return 2
|
||||
secret: Final = environ.get(SECRET_VARIABLE, "")
|
||||
root: Final = Path(argv[0])
|
||||
if not secret:
|
||||
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
|
||||
return 0
|
||||
if not root.is_dir():
|
||||
print(f"{root} is not a directory, nothing to scan", file=out)
|
||||
return 0
|
||||
try:
|
||||
scanned, hits = scan(root, secret)
|
||||
except Exception as error:
|
||||
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
|
||||
return 1
|
||||
for hit in hits:
|
||||
print(f"secret found: {hit.location} ({hit.form})", file=out)
|
||||
if hits:
|
||||
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
|
||||
return 1
|
||||
print(f"no copy of the secret in {scanned} files under {root}", file=out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))
|
||||
10
.github/workflows/cost-map-guard.yml
vendored
10
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -8,6 +8,7 @@ on:
|
|||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
|
|
@ -21,16 +22,19 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
- name: Check out the base branch tip and fetch the pull request head and its merge base
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
git checkout -q --detach "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
|
|
|
|||
2
.github/workflows/image-scan.yml
vendored
2
.github/workflows/image-scan.yml
vendored
|
|
@ -13,8 +13,10 @@ on:
|
|||
- migrations/Dockerfile
|
||||
- migrations/run.py
|
||||
- gateway/Dockerfile
|
||||
- gateway/launch.py
|
||||
- gateway/main.py
|
||||
- gateway/routes/allowlist.py
|
||||
- litellm_cli/**
|
||||
- backend/Dockerfile
|
||||
- backend/main.py
|
||||
- litellm-rust/**
|
||||
|
|
|
|||
152
.github/workflows/lens-install-smoke.yml
vendored
152
.github/workflows/lens-install-smoke.yml
vendored
|
|
@ -1,152 +0,0 @@
|
|||
name: Lens installation smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
lens_image:
|
||||
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
lens_upgrade_image:
|
||||
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
gateway_baseline_ref:
|
||||
description: Previously qualified gateway source commit, full SHA
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lens-install-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
- component: ui
|
||||
dockerfile: ui/Dockerfile
|
||||
- component: migrations
|
||||
dockerfile: migrations/Dockerfile
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
baseline: true
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
baseline: true
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
baseline: true
|
||||
env:
|
||||
COMPONENT: ${{ matrix.component }}
|
||||
DOCKERFILE: ${{ matrix.dockerfile }}
|
||||
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
|
||||
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
|
||||
CURRENT_REF: ${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.SOURCE_REF }}
|
||||
- name: Verify the selected source
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
|
||||
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
|
||||
test "$SOURCE_REF" != "$CURRENT_REF"
|
||||
fi
|
||||
- name: Build the matching release image
|
||||
run: |
|
||||
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
|
||||
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
|
||||
- name: Save the matching release image
|
||||
run: |
|
||||
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
|
||||
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
|
||||
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
|
||||
compression-level: 0
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
|
||||
helm-install:
|
||||
needs: build-images
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: lens-install-*-${{ github.sha }}
|
||||
merge-multiple: true
|
||||
path: ${{ runner.temp }}/lens-install-images
|
||||
- name: Load the matching release images
|
||||
run: |
|
||||
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
|
||||
gzip -dc "$archive" | docker load
|
||||
rm "$archive"
|
||||
done
|
||||
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
|
||||
with:
|
||||
cosign-release: v2.4.3
|
||||
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ vars.LENS_READ_USER || github.actor }}
|
||||
password: ${{ secrets.LENS_READ_TOKEN }}
|
||||
- name: Verify independent Lens images before loading them
|
||||
env:
|
||||
LENS_IMAGE: ${{ inputs.lens_image }}
|
||||
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
|
||||
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
|
||||
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
|
||||
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
|
||||
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
|
||||
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
|
||||
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
|
||||
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
|
||||
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
|
||||
docker pull "$image"
|
||||
done
|
||||
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
|
||||
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
|
||||
- name: Install pinned Kubernetes test tools
|
||||
run: |
|
||||
curl --fail --location --output "$RUNNER_TEMP/kind" \
|
||||
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
|
||||
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kind"
|
||||
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
|
||||
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
|
||||
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kubectl"
|
||||
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
|
||||
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
|
||||
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
|
||||
- name: Install, ingest, upgrade, and restart both charts
|
||||
run: bash tests/e2e/migrations/lens_helm_smoke.sh
|
||||
6
.github/workflows/spend-logs-guard.yml
vendored
6
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -19,17 +19,17 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
|
|
|
|||
199
.github/workflows/test-integration.yml
vendored
Normal file
199
.github/workflows/test-integration.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
name: integration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
schedule:
|
||||
- cron: "5 8-20/3 * * 1-6"
|
||||
timezone: America/Los_Angeles
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
suite:
|
||||
description: Integration group to run
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- providers
|
||||
- extensions
|
||||
- accounting
|
||||
- management
|
||||
- management-replica
|
||||
- database
|
||||
- database-replica
|
||||
- mcp
|
||||
- security
|
||||
- cost
|
||||
- sdk
|
||||
- browser
|
||||
default: all
|
||||
workers:
|
||||
description: Workers per group, blank keeps each group's default
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
integration:
|
||||
name: integration ${{ matrix.suite }}
|
||||
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
|
||||
environment: integration-license
|
||||
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: circle_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--shm-size 1g
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
|
||||
|
||||
- name: Install redis-server 7.2.16
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/redis-bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
|
||||
docker rm "$cid"
|
||||
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
|
||||
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install MCP SDK1 peer
|
||||
if: matrix.suite == 'mcp'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: |
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
tests/e2e/ui/package-lock.json
|
||||
|
||||
- name: Install browser dependencies and build the dashboard
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
(cd ui/litellm-dashboard && npm ci)
|
||||
(cd tests/e2e/ui && npm ci)
|
||||
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
|
||||
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
|
||||
|
||||
- name: Run integration contracts
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SUITE: ${{ matrix.suite }}
|
||||
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
|
||||
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: |
|
||||
case "$SUITE" in
|
||||
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
|
||||
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
|
||||
esac
|
||||
|
||||
- name: Verify every case ran exactly once
|
||||
if: ${{ !cancelled() && matrix.suite != 'browser' }}
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
|
||||
test -n "$record"
|
||||
mkdir -p "$RUNNER_TEMP/shards/0"
|
||||
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
|
||||
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
|
||||
|
||||
- name: Report MCP coverage
|
||||
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
coverage_dir=test-results/integration-mcp/coverage
|
||||
test -f "$coverage_dir/coverage.txt" && exit 0
|
||||
export COVERAGE_FILE="$PWD/$coverage_dir/data"
|
||||
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
|
||||
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
|
||||
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
|
||||
tail -n 1 "$coverage_dir/coverage.txt"
|
||||
|
||||
- name: Check results for the license
|
||||
id: secret-scan
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: python .github/scripts/assert_secret_absent.py test-results
|
||||
|
||||
- name: Upload results
|
||||
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
|
||||
timeout-minutes: 10
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: test-results/
|
||||
retention-days: 14
|
||||
25
.github/workflows/test-linting.yml
vendored
25
.github/workflows/test-linting.yml
vendored
|
|
@ -48,11 +48,13 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$MERGE_BASE"
|
||||
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
|
||||
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
|
||||
|
|
@ -264,6 +266,7 @@ jobs:
|
|||
name: ui
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
|
|
@ -281,16 +284,12 @@ jobs:
|
|||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
|
|
@ -630,6 +629,12 @@ jobs:
|
|||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_no_none_string_error_fields
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
|
||||
|
||||
- name: test_check_no_none_string_error_fields
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
|
|
|
|||
17
.github/workflows/test-unit.yml
vendored
17
.github/workflows/test-unit.yml
vendored
|
|
@ -245,6 +245,7 @@ jobs:
|
|||
tests/unit/llms/chatgpt
|
||||
tests/unit/llms/clarifai
|
||||
tests/unit/llms/claude_code
|
||||
tests/unit/llms/clf_ai_gateway
|
||||
tests/unit/llms/cloudflare
|
||||
tests/unit/llms/codestral
|
||||
tests/unit/llms/codex
|
||||
|
|
@ -1003,9 +1004,9 @@ jobs:
|
|||
run: |
|
||||
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
|
|
@ -1150,18 +1151,20 @@ jobs:
|
|||
env:
|
||||
CI: "true"
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
|
||||
|
||||
if [ -z "$BASE_SHA" ]; then
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "Push to $GITHUB_REF_NAME: running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
changed_files=()
|
||||
|
|
@ -1451,9 +1454,9 @@ jobs:
|
|||
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
run: |
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -153,7 +153,7 @@ lint-install:
|
|||
# the same set CI's ':(glob)litellm/**/*.py' selects.
|
||||
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
@base_ref=$$($(RESOLVE_BASE)) && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
|
||||
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
|
||||
if [ -z "$$files" ]; then \
|
||||
echo "No changed litellm Python files to format-check."; \
|
||||
|
|
|
|||
52
README.md
52
README.md
|
|
@ -141,12 +141,56 @@ response = client.chat.completions.create(
|
|||
)
|
||||
```
|
||||
|
||||
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Auto Routing</b> - Choose a model for each prompt (Python SDK + AI Gateway)</summary>
|
||||
|
||||
### Python SDK
|
||||
|
||||
[**Docs: Autorouter SDK**](https://docs.litellm.ai/docs/autorouter_sdk)
|
||||
|
||||
Install with `uv add litellm` and set `OPENAI_API_KEY` for the models below.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from litellm import Router
|
||||
|
||||
router = Router(model_list=[
|
||||
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-6-luna"}},
|
||||
{"model_name": "capable", "litellm_params": {"model": "openai/gpt-6.1-sol"}},
|
||||
{"model_name": "auto", "litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "heuristic",
|
||||
"tiers": {"SIMPLE": "fast", "MEDIUM": "fast",
|
||||
"COMPLEX": "capable", "REASONING": "capable"},
|
||||
"return_raw_model_name": True,
|
||||
},
|
||||
"complexity_router_default_model": "capable",
|
||||
}},
|
||||
])
|
||||
response = asyncio.run(router.acompletion(
|
||||
model="auto", messages=[{"role": "user", "content": "What is 2+2?"}],
|
||||
))
|
||||
print(response.model, response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### AI Gateway (Proxy Server)
|
||||
|
||||
[**Create an Auto Router**](https://docs.litellm.ai/docs/auto_router/setup), then call its model name. Replace `auto` and the URL below with yours, and set `LITELLM_API_KEY` to your virtual key.
|
||||
|
||||
```shell
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"auto","messages":[{"role":"user","content":"What is 2+2?"}]}'
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
|
||||
<summary><b>A2A</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
|
||||
|
||||
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
|
||||
|
||||
|
|
@ -335,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
@ -354,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|
||||
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
|
||||
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
|
||||
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
|
||||
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
@ -442,7 +488,7 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
|
|||
<tr>
|
||||
<th style={{width: '14%'}}></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/python_sdk">LiteLLM Python SDK</a></strong></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
|
|
|||
|
|
@ -398,7 +398,7 @@ class CheckBatchCost:
|
|||
deployment. Once that deployment leaves the router, default fallbacks can
|
||||
silently send the retrieve to a provider that never saw the batch, so its
|
||||
404 must not retire a potentially recoverable row."""
|
||||
return self.llm_router.get_deployment(model_id=model_id) is not None
|
||||
return self.llm_router.registry().deployment(model_id) is not None
|
||||
|
||||
@staticmethod
|
||||
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
|
||||
|
|
@ -564,7 +564,7 @@ class CheckBatchCost:
|
|||
def _get_deployment_id_for_bare_model(
|
||||
self, bare_model_name: str, llm_provider: str
|
||||
) -> Optional[str]:
|
||||
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
|
||||
model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
|
||||
deployment_id = (
|
||||
self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
|
||||
)
|
||||
|
|
@ -580,7 +580,7 @@ class CheckBatchCost:
|
|||
) -> Optional[str]:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
for deployment in self.llm_router.get_model_list(model_name=None) or []:
|
||||
for deployment in self.llm_router.registry().deployments() or []:
|
||||
litellm_params = deployment.get("litellm_params") or {}
|
||||
actual_model = litellm_params.get("model")
|
||||
if not isinstance(actual_model, str):
|
||||
|
|
@ -623,8 +623,9 @@ class CheckBatchCost:
|
|||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
|
||||
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
registry = self.llm_router.registry()
|
||||
for deployment_id in registry.names(model_group, kind="ids"):
|
||||
deployment_info = registry.deployment(deployment_id)
|
||||
if deployment_info is None:
|
||||
continue
|
||||
try:
|
||||
|
|
@ -743,7 +744,7 @@ class CheckBatchCost:
|
|||
except (IndexError, AttributeError):
|
||||
pass
|
||||
|
||||
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=cast(str, raw_output_file_id),
|
||||
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
|
||||
|
|
@ -766,7 +767,7 @@ class CheckBatchCost:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
deployment_info = self.llm_router.get_deployment(model_id=model_id)
|
||||
deployment_info = self.llm_router.registry().deployment(model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.info(
|
||||
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
|
||||
|
|
@ -996,7 +997,7 @@ class CheckBatchCost:
|
|||
)
|
||||
|
||||
try:
|
||||
deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
|
||||
deployment_info: Final = self.llm_router.registry().deployment(model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
|
||||
|
|
@ -1009,7 +1010,7 @@ class CheckBatchCost:
|
|||
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
|
||||
)
|
||||
cost_provider: Final = cast(BatchFileProvider, llm_provider)
|
||||
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
file_access: Final = {
|
||||
**credentials,
|
||||
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ class CheckResponsesCost:
|
|||
|
||||
def _resolve_deployment(self, response_id: str) -> Deployment | None:
|
||||
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
|
||||
return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
|
||||
return self.llm_router.registry().deployment(model_id) if model_id is not None else None
|
||||
|
||||
async def _get_response(
|
||||
self,
|
||||
|
|
@ -415,7 +415,7 @@ class CheckResponsesCost:
|
|||
if target.deployment is None or not isinstance(requested, str):
|
||||
return MappingProxyType({})
|
||||
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
|
||||
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
|
||||
self.llm_router.registry().deployments(requested, team_id=team_id) or ()
|
||||
)
|
||||
serving_id: Final = target.deployment.model_info.id
|
||||
return MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
|
|||
if llm_router is None or model_id is None:
|
||||
return None
|
||||
try:
|
||||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
|
||||
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
except Exception as error:
|
||||
verbose_logger.warning(
|
||||
"Failed to retrieve credentials for provider file "
|
||||
|
|
@ -2650,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
|
||||
}
|
||||
for model_id, model_file_id in specific_model_file_id_mapping.items():
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
delete_data = {
|
||||
**filtered_data,
|
||||
**(
|
||||
|
|
@ -2694,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# against the deployment's configured bucket, which they only
|
||||
# trust from this immutable server-side snapshot, never from
|
||||
# request params.
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
if credentials is not None:
|
||||
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -290,7 +290,7 @@ def _check_team_project_limits(
|
|||
|
||||
|
||||
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
|
||||
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
|
||||
return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
|
||||
|
||||
|
||||
def _project_models_expanding_at_request_time(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
|||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
|
|||
wins under token auth too, so exporting it here is enough for every worker to
|
||||
pick the pooled URL up unchanged.
|
||||
|
||||
LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
|
||||
before it can answer its first 5 s health ping, so a slow host would kill each one.
|
||||
Workers then load the gateway app through ``worker_app`` without garbage collection
|
||||
pauses, which hold the GIL long enough to miss a ping.
|
||||
|
||||
Run with:
|
||||
python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
|
||||
"""
|
||||
|
|
@ -15,19 +20,19 @@ Run with:
|
|||
import os
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
export_pooled_database_url,
|
||||
start_in_container_pgbouncer,
|
||||
)
|
||||
from litellm_cli.main import without_gc_pauses
|
||||
|
||||
GATEWAY_APP: Final = "gateway.main:app"
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
|
||||
|
||||
GATEWAY_APP: Final = "gateway.launch:worker_app"
|
||||
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
|
||||
|
||||
|
||||
|
|
@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
|
|||
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
|
||||
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
|
||||
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
|
||||
return (GATEWAY_APP, *argv)
|
||||
return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
|
||||
return (GATEWAY_APP, "--factory", *argv)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
|
||||
|
||||
|
||||
def pool_database_url(
|
||||
settings: DatabaseURLSettings,
|
||||
pgbouncer: PgBouncerSettings,
|
||||
settings: "DatabaseURLSettings",
|
||||
pgbouncer: "PgBouncerSettings",
|
||||
environ: Mapping[str, str],
|
||||
) -> str | PgBouncerError | None:
|
||||
) -> "str | PgBouncerError | None":
|
||||
"""Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
|
||||
|
||||
The upstream URL is whatever ``apply_to_env`` assembled from the discrete
|
||||
``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
|
||||
auth the pooler mints and renews the upstream token itself.
|
||||
"""
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
|
||||
|
||||
if not pgbouncer.enabled:
|
||||
return None
|
||||
upstream_url: Final = environ.get("DATABASE_URL")
|
||||
|
|
@ -58,11 +65,20 @@ def pool_database_url(
|
|||
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
|
||||
|
||||
|
||||
def worker_app() -> "FastAPI":
|
||||
with without_gc_pauses():
|
||||
from gateway.main import app
|
||||
return app
|
||||
|
||||
|
||||
def _serve(argv: Sequence[str]) -> None:
|
||||
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
|
||||
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
settings.apply_to_env()
|
||||
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
|||
"/test",
|
||||
"/debug/memory/summary",
|
||||
"/api/event_logging/batch",
|
||||
"/eu_jev/v1/systemone",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,5 @@ dependencies:
|
|||
- name: redis
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
version: 18.19.1
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
|
||||
generated: '2026-10-09T04:18:10.643228000Z'
|
||||
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
|
||||
generated: '2026-10-30T02:13:11.505726000Z'
|
||||
|
|
|
|||
|
|
@ -39,6 +39,3 @@ dependencies:
|
|||
version: "18.19.1"
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
condition: redis.enabled
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -256,9 +256,7 @@ itself.
|
|||
|
||||
## Connect Lens
|
||||
|
||||
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
## Shutdown accounting
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
|
|||
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -402,95 +398,28 @@ shutdown drain window.
|
|||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.labels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
|
||||
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
{{- if .Values.ingress.enabled -}}
|
||||
{{- $fullName := include "litellm.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
{{- $lensMode := include "litellm.lens.mode" . -}}
|
||||
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
|
||||
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
|
||||
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
|
||||
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
|
||||
|
|
@ -46,18 +44,18 @@ spec:
|
|||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
|
||||
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
name: {{ $.Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ $.Values.lensWorker.service.port }}
|
||||
number: {{ $.Values.lens.service.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
|
||||
servicePort: {{ $.Values.lensWorker.service.port }}
|
||||
serviceName: {{ $.Values.lens.service.name }}
|
||||
servicePort: {{ $.Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .paths }}
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
128
helm/litellm-helm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
- ingress.yaml
|
||||
tests:
|
||||
- it: connects the proxy to an external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service ahead of the host paths
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[0].path
|
||||
value: /lens-ingest
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- deployment.yaml
|
||||
- configmap-litellm.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- ingress.yaml
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects deployment.yaml to the shared Lens service
|
||||
template: deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.hosts:
|
||||
- host: gateway.example
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in deployment.yaml
|
||||
template: deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
|
@ -668,70 +668,15 @@ serviceMonitor:
|
|||
# - test-namespace
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
dependencies:
|
||||
- name: lens
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
version: 0.1.0-dev.0
|
||||
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
|
||||
generated: '2026-10-09T04:18:10.642706000Z'
|
||||
|
|
@ -6,8 +6,3 @@ version: 0.1.0
|
|||
appVersion: "0.1.0"
|
||||
annotations:
|
||||
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
|
||||
|
||||
dependencies:
|
||||
- name: lens
|
||||
version: 0.1.0-dev.0
|
||||
repository: oci://ghcr.io/berriai/charts
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
# LiteLLM componentized chart
|
||||
|
||||
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
|
||||
This chart deploys separate gateway, backend and UI services
|
||||
|
||||
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
|
||||
|
||||
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
|
|||
collector containers through an emptyDir. Empty when the sidecar is off
|
||||
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
|
||||
*/}}
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
@ -635,93 +631,27 @@ shutdown drain window.
|
|||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensConnectionEnv" -}}
|
||||
{{- $mode := include "litellm.lens.mode" . -}}
|
||||
{{- if ne $mode "disabled" }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
|
||||
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
|
||||
{{- with .Values.lensWorker }}
|
||||
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
|
||||
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
|
||||
{{- end }}
|
||||
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
|
||||
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- with .Values.lens }}
|
||||
{{- if .enabled }}
|
||||
- name: LITELLM_LENS_URL
|
||||
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
|
||||
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
|
||||
- name: LITELLM_LENS_PUBLIC_URL
|
||||
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
|
||||
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
|
||||
- name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
|
||||
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
|
||||
key: {{ .gatewaySecret.key | quote }}
|
||||
- name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
|
||||
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
|
||||
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
|
||||
key: {{ .serviceTokenSecret.key | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.labels" -}}
|
||||
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
|
||||
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- toYaml $labels -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
|
||||
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.gatewaySecretName" -}}
|
||||
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.bundledClickhouse" -}}
|
||||
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.publicUrl" -}}
|
||||
{{- if .Values.lensWorker.publicUrl -}}
|
||||
{{- .Values.lensWorker.publicUrl -}}
|
||||
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
|
||||
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
|
||||
{{- else if .Values.lensWorker.ingress.enabled -}}
|
||||
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
|
||||
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
|
||||
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
|
||||
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
|
||||
{{- else -}}
|
||||
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lensWorker.clickhouseName" -}}
|
||||
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.mode" -}}
|
||||
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
|
||||
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
|
||||
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
|
||||
{{- end -}}
|
||||
{{- $mode -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.lens.render" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
|
||||
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
|
||||
{{- $_ := set $values "component" "lens-worker" -}}
|
||||
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
|
||||
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
|
||||
{{- $_ := set $values.gateway "enabled" true -}}
|
||||
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
|
||||
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
|
||||
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
|
||||
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
|
||||
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
|
||||
{{- $_ := set $values "ingestionUrl" $ingestion -}}
|
||||
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
|
||||
{{- end -}}
|
||||
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -166,14 +166,14 @@ spec:
|
|||
- host: {{ $componentHosts.gateway | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
|
|
@ -249,15 +249,15 @@ spec:
|
|||
port:
|
||||
number: {{ $gatewayPort }}
|
||||
{{- end }}
|
||||
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
|
||||
{{- if and .Values.lens.enabled .Values.lens.service.name }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
|
||||
- path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
|
||||
name: {{ .Values.lens.service.name }}
|
||||
port:
|
||||
number: {{ .Values.lensWorker.service.port }}
|
||||
number: {{ .Values.lens.service.port }}
|
||||
{{- end }}
|
||||
{{- /*
|
||||
--- Operator-supplied extra paths (ingress.extraPaths) ---
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
|
||||
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
|
||||
{{- end }}
|
||||
|
|
@ -15,7 +15,6 @@ tests:
|
|||
ingress.componentHosts.ui: ui.example.com
|
||||
ingress.componentHosts.backend: backend.example.com
|
||||
ingress.componentHosts.gateway: gateway.example.com
|
||||
lensWorker.mode: disabled
|
||||
ingress.tls:
|
||||
- hosts:
|
||||
- ui.example.com
|
||||
|
|
@ -83,12 +82,12 @@ tests:
|
|||
ui.backendUrl: https://backend.example.com
|
||||
ui.publicUrl: https://ui.example.com
|
||||
backend.gatewayUrl: https://gateway.example.com
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.externalUrl: https://lens.example.com
|
||||
lensWorker.publicUrl: https://lens.example.com
|
||||
lensWorker.serviceTokenSecret.name: existing-lens-signing
|
||||
lensWorker.gateway.secretName: existing-lens-signing
|
||||
lens.enabled: true
|
||||
lens.service.name: existing-lens
|
||||
lens.url: http://existing-lens:4318
|
||||
lens.publicUrl: https://gateway.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: existing-lens-signing
|
||||
lens.gatewaySecret.name: existing-lens-signing
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[2].http.paths[0].path
|
||||
|
|
|
|||
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
166
helm/litellm/tests/lens_connection_tests.yaml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
suite: lens connection
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: connects the gateway to an external Lens
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: connects the backend to an external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.lens.svc:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://lens.example.com/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-gateway
|
||||
key: gateway-secret
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-service
|
||||
key: service-token
|
||||
|
||||
- it: requires lens.url when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.url is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.publicUrl when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.publicUrl is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.gatewaySecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
|
||||
|
||||
- it: requires lens.serviceTokenSecret.name when Lens is enabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
|
||||
|
||||
- it: routes /lens-ingest to the named Lens service on the single ingress host
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
lens.service.name: existing-lens
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port:
|
||||
number: 4318
|
||||
|
||||
- it: leaves /lens-ingest off the ingress without a Lens service name
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.host: litellm.example.com
|
||||
lens.enabled: true
|
||||
lens.url: http://lens.lens.svc:4318
|
||||
lens.publicUrl: https://lens.example.com/lens-ingest
|
||||
lens.gatewaySecret.name: lens-gateway
|
||||
lens.serviceTokenSecret.name: lens-service
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.rules[0].http.paths
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
any: true
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
suite: Lens endpoint defaults
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
ingress.tls:
|
||||
- hosts: [gateway.example]
|
||||
secretName: tls
|
||||
tests:
|
||||
- it: derives the trace endpoint from the deployment hostname and TLS
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- it: preserves an explicitly configured public address
|
||||
set:
|
||||
lensWorker.publicUrl: https://custom.example/prefix
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://custom.example/prefix
|
||||
- it: uses the dedicated Lens ingress when configured
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls:
|
||||
- hosts: [traces.example]
|
||||
secretName: traces-tls
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.className: alb
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations:
|
||||
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
|
||||
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
|
||||
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://traces.example
|
||||
- it: uses HTTP for a dedicated ingress without its own TLS
|
||||
set:
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
lensWorker.ingress.tls: []
|
||||
lensWorker.ingress.annotations: {}
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://traces.example
|
||||
- it: uses HTTP when ingress has no TLS
|
||||
set:
|
||||
ingress.tls: []
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: http://gateway.example/lens-ingest
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment modes
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
- ingress.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: retains the existing deployment before transferring its release ownership
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.retainResources: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: keeps the ingestion endpoint on an independently deployed Lens service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
ingress.enabled: true
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: existing-lens
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: connects an external deployment without installing Lens resources
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/service.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: sends delegated requests to the explicitly selected external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens.other-namespace:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: external-lens-signing
|
||||
key: gateway-secret
|
||||
- it: disables Lens explicitly even with a retained legacy enable flag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: bundles Lens through explicit mode selection
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_MODE
|
||||
value: standalone
|
||||
- it: uses separate credentials for external identity and service access
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: worker-service, key: token}
|
||||
- it: supplies the same separate signing secret to the bundled service
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: delegated-identity, key: signature}
|
||||
- it: routes explicit bundled mode with the enable flag off
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: bundled
|
||||
lensWorker.enabled: false
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: gateway-lens-worker
|
||||
port: {number: 4318}
|
||||
- it: routes external mode to its selected service
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: "existing-lens"
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: existing-lens
|
||||
port: {number: 4318}
|
||||
- it: omits ingestion when an external service has no ingress backend
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: external
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
- it: omits ingestion for disabled mode with the enable flag on
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: gateway
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
lensWorker.mode: disabled
|
||||
lensWorker.enabled: true
|
||||
lensWorker.externalUrl: http://existing-lens:4318
|
||||
lensWorker.externalServiceName: ""
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.rules[0].http.paths
|
||||
content: {path: /lens-ingest}
|
||||
any: true
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
---
|
||||
suite: Lens deployment mode validation
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
values:
|
||||
- "./values/required.yaml"
|
||||
set:
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
tests:
|
||||
- it: requires a separate identity signing reference for external Lens
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: worker-service
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.gateway.secretName is required for external Lens
|
||||
- it: requires an external service address
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.externalUrl is required for external Lens
|
||||
- it: requires the shared server credential for an external service
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
|
||||
- it: requires an agent reachable URL for external ingestion
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: external
|
||||
lensWorker.externalUrl: http://lens.other-namespace:4318
|
||||
lensWorker.serviceTokenSecret.name: external-lens-signing
|
||||
lensWorker.gateway.secretName: external-lens-signing
|
||||
lensWorker.publicUrl: ''
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required for external Lens
|
||||
- it: rejects unsupported deployment modes
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.mode: typo
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.mode must be bundled, external or disabled
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
suite: Lens preserves credentials across Helm upgrades
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
upgrade: true
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
v1/Secret:
|
||||
gvr:
|
||||
version: v1
|
||||
resource: secrets
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-gateway
|
||||
namespace: lens
|
||||
data:
|
||||
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-service
|
||||
namespace: lens
|
||||
data:
|
||||
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: lens-test-lens-clickhouse
|
||||
namespace: lens
|
||||
data:
|
||||
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
tests:
|
||||
- it: preserves the separate signing credential across upgrades
|
||||
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.gateway-secret
|
||||
value: c2F2ZWQtc2lnbmluZy1rZXk=
|
||||
- notExists: {path: data.service-token}
|
||||
- it: reuses the service credential instead of breaking running services
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.service-token
|
||||
value: c2F2ZWQtc2VydmljZS10b2tlbg==
|
||||
- it: reuses the database password instead of locking out stored traces
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.password
|
||||
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
suite: Lens service isolation and ingestion routing
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- gateway/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- ingress.yaml
|
||||
- lens/ingress.yaml
|
||||
- lens/service.yaml
|
||||
- lens/deployment.yaml
|
||||
tests:
|
||||
- it: connects gateway/deployment.yaml to the shared Lens service
|
||||
template: gateway/deployment.yaml
|
||||
set: &id001
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: connects backend/deployment.yaml to the shared Lens service
|
||||
template: backend/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
value: http://lens-test-lens-worker:4318
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_PUBLIC_URL
|
||||
value: https://gateway.example/lens-ingest
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-gateway
|
||||
key: gateway-secret
|
||||
- it: routes uploads directly to Lens instead of the gateway
|
||||
template: ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
ingress.enabled: true
|
||||
ingress.host: gateway.example
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /lens-ingest
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
number: 4318
|
||||
- it: keeps internal routes out of a dedicated ingestion hostname
|
||||
template: lens/ingress.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://gateway.example/lens-ingest
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.ingress.enabled: true
|
||||
lensWorker.ingress.host: traces.example
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths
|
||||
value:
|
||||
- path: /v1/
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: lens-test-lens-worker
|
||||
port:
|
||||
name: otlp
|
||||
- it: maps the Lens service to the ingestion listener
|
||||
template: lens/service.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: lens-worker
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- name: otlp
|
||||
port: 4318
|
||||
targetPort: otlp
|
||||
- it: gives only Lens the ClickHouse secret
|
||||
template: lens/deployment.yaml
|
||||
set: *id001
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-storage
|
||||
key: url
|
||||
- it: requires an agent reachable ingestion URL
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: ''
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
|
||||
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: omits Lens connection settings when disabled in backend/deployment.yaml
|
||||
template: backend/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_LENS_URL
|
||||
any: true
|
||||
- it: preserves an existing ClickHouse database and retention
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
lensWorker.clickhouseDatabase: existing_traces
|
||||
lensWorker.retentionDays: 45
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_DATABASE
|
||||
value: existing_traces
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: AGENT_TRACING_RETENTION_DAYS
|
||||
value: '45'
|
||||
- it: keeps Lens pods outside the gateway autoscaling selector
|
||||
template: lens/deployment.yaml
|
||||
set: &id002
|
||||
nameOverride: inference
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: inference-lens-worker
|
||||
- it: preserves the existing gateway deployment selector
|
||||
template: gateway/deployment.yaml
|
||||
set: *id002
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["app.kubernetes.io/name"]
|
||||
value: inference
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,164 +0,0 @@
|
|||
suite: Lens managed setup
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
release:
|
||||
namespace: lens
|
||||
name: lens-test
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- lens/clickhouse.yaml
|
||||
- lens/secrets.yaml
|
||||
tests:
|
||||
- it: generates private login, gateway and storage credentials
|
||||
template: lens/secrets.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 4
|
||||
- matchRegex:
|
||||
path: data.service-token
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: '^[A-Za-z0-9+/]{86}==$'
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/resource-policy"]
|
||||
value: keep
|
||||
- it: connects Lens to its private bundled storage
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: lens-test-lens-clickhouse
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: stores traces on a persistent volume with the chosen storage class
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
lensWorker.clickhouse.storage: 40Gi
|
||||
lensWorker.clickhouse.storageClassName: fast
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.volumeClaimTemplates[0].spec
|
||||
value:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: fast
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: lens-test-lens-clickhouse
|
||||
key: password
|
||||
- it: keeps an existing external database instead of creating a new one
|
||||
template: lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: leaves supplied secrets under their existing manager
|
||||
template: lens/secrets.yaml
|
||||
set:
|
||||
lensWorker.clickhouseSecret.name: external-clickhouse
|
||||
lensWorker.serviceTokenSecret.name: external-service
|
||||
lensWorker.adminTokenSecret.name: external-admin
|
||||
lensWorker.gateway.secretName: external-signing
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: creates no credentials or database when Lens is disabled
|
||||
templates:
|
||||
- lens/secrets.yaml
|
||||
- lens/clickhouse.yaml
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires an external database when bundled storage is explicitly disabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.clickhouse.enabled: false
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
|
||||
- it: keeps storage names valid and uses the same name for the connection
|
||||
set:
|
||||
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 0
|
||||
- equal:
|
||||
path: spec.serviceName
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/secrets.yaml
|
||||
documentIndex: 1
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_HOST
|
||||
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
template: lens/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
|
||||
key: password
|
||||
template: lens/deployment.yaml
|
||||
- it: allows cold storage startup and inherits registry credentials
|
||||
template: lens/clickhouse.yaml
|
||||
documentIndex: 1
|
||||
set:
|
||||
imagePullSecrets: [{name: registry-auth}]
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.imagePullSecrets
|
||||
value: [{name: registry-auth}]
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].startupProbe
|
||||
value:
|
||||
httpGet: {path: /ping, port: http}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 60
|
||||
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
suite: Independent Lens release and credentials
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
set:
|
||||
fullnameOverride: lens-test
|
||||
lensWorker.enabled: true
|
||||
lensWorker.publicUrl: https://traces.example
|
||||
lensWorker.serviceTokenSecret.name: lens-service
|
||||
lensWorker.clickhouseSecret.name: lens-storage
|
||||
tests:
|
||||
- it: selects a Lens release independently of the gateway
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
lensWorker.image.tag: 2.3.4
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:2.3.4
|
||||
- it: retains the pinned Lens chart default when only the gateway changes
|
||||
chart:
|
||||
appVersion: 9.8.7
|
||||
set:
|
||||
backend.image.tag: gateway-9.8.7
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens:0.1.0-dev.0
|
||||
- it: selects development publications explicitly
|
||||
set:
|
||||
lensWorker.image.repository: ghcr.io/berriai/lens-dev
|
||||
lensWorker.image.tag: sha-0123456789abcdef
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
|
||||
- it: preserves a verified registry mirror and digest
|
||||
set:
|
||||
lensWorker.image.repository: registry.example/lens
|
||||
lensWorker.image.tag: replaced-label
|
||||
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
- it: rejects a malformed digest
|
||||
set:
|
||||
lensWorker.image.digest: sha256:invalid
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
|
||||
- it: keeps the legacy enablement switch working
|
||||
set:
|
||||
lensWorker.enabled: false
|
||||
asserts:
|
||||
- hasDocuments: {count: 0}
|
||||
- it: keeps isolated local analysis with delegated gateway access
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: LENS_MODE, value: standalone}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_GATEWAY_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].emptyDir
|
||||
value: {medium: Memory, sizeLimit: 1Gi}
|
||||
|
|
@ -654,70 +654,15 @@ ui:
|
|||
topologySpreadConstraints: []
|
||||
|
||||
lens:
|
||||
library: true
|
||||
|
||||
lensWorker:
|
||||
retainResources: false
|
||||
mode: ""
|
||||
externalUrl: ""
|
||||
externalServiceName: ""
|
||||
gateway:
|
||||
secretName: ""
|
||||
secretKey: gateway-secret
|
||||
standaloneUrl: ""
|
||||
adminTokenSecret:
|
||||
name: ""
|
||||
key: admin-token
|
||||
extraEnv: []
|
||||
extraEnvFrom: []
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/lens
|
||||
tag: "0.1.0-dev.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
url: ""
|
||||
publicUrl: ""
|
||||
service:
|
||||
name: ""
|
||||
key: token
|
||||
port: 4318
|
||||
gatewaySecret:
|
||||
name: ""
|
||||
key: gateway-secret
|
||||
serviceTokenSecret:
|
||||
name: ""
|
||||
key: service-token
|
||||
clickhouse:
|
||||
enabled: true
|
||||
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
|
||||
storage: 20Gi
|
||||
storageClassName: null
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
clickhouseDatabase: litellm
|
||||
retentionDays: 14
|
||||
clickhouseSecret:
|
||||
name: ""
|
||||
key: url
|
||||
publicUrl: ""
|
||||
service:
|
||||
port: 4318
|
||||
annotations: {}
|
||||
ingress:
|
||||
enabled: false
|
||||
path: /v1/
|
||||
className: ""
|
||||
host: ""
|
||||
annotations: {}
|
||||
tls: []
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -238,7 +238,8 @@ impl AnthropicBeta {
|
|||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
| Self::ThinkingBindingControls20260801
|
||||
| Self::ThinkingDisplayUpdates20260818 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -275,7 +276,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
BetaProvider::AzureAi,
|
||||
|
|
|
|||
|
|
@ -520,7 +520,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
|
|||
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
|
||||
disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars
|
||||
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
|
||||
http2: bool = False
|
||||
http2: bool | None = None
|
||||
network_mock: bool = False # When True, use mock transport — no real network calls
|
||||
|
||||
####### STOP SEQUENCE LIMIT #######
|
||||
|
|
@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
|
|||
baseten_models: Set[str] = set()
|
||||
fireworks_ai_embedding_models: Set = set()
|
||||
deepinfra_models: Set = set()
|
||||
clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets
|
||||
perplexity_models: Set = set()
|
||||
watsonx_models: Set = set()
|
||||
gemini_models: Set = set()
|
||||
|
|
@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
bedrock_converse_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepinfra":
|
||||
deepinfra_models.add(key)
|
||||
elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map
|
||||
clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key
|
||||
elif value.get("litellm_provider") == "perplexity":
|
||||
perplexity_models.add(key)
|
||||
elif value.get("litellm_provider") == "watsonx":
|
||||
|
|
@ -1110,6 +1113,7 @@ model_list = list(
|
|||
| set(ollama_models)
|
||||
| bedrock_models
|
||||
| deepinfra_models
|
||||
| clf_ai_gateway_models
|
||||
| perplexity_models
|
||||
| set(maritalk_models)
|
||||
| runwayml_models
|
||||
|
|
@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict:
|
|||
"ollama": ollama_models,
|
||||
"ollama_chat": ollama_models,
|
||||
"deepinfra": deepinfra_models,
|
||||
"clf_ai_gateway": clf_ai_gateway_models,
|
||||
"perplexity": perplexity_models,
|
||||
"maritalk": maritalk_models,
|
||||
"watsonx": watsonx_models,
|
||||
|
|
@ -1783,6 +1788,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bespoke.decisions.transformation import (
|
||||
BespokeDecisionsConfig as BespokeDecisionsConfig,
|
||||
)
|
||||
from .llms.eu_jev.decisions.transformation import (
|
||||
EuJevDecisionsConfig as EuJevDecisionsConfig,
|
||||
)
|
||||
from .llms.openai.decisions.transformation import (
|
||||
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
|
||||
)
|
||||
|
|
@ -2108,6 +2116,9 @@ if TYPE_CHECKING:
|
|||
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
|
||||
)
|
||||
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
|
||||
from .llms.clf_ai_gateway.chat.transformation import (
|
||||
ClfAiGatewayConfig as _ClfAiGatewayConfig,
|
||||
)
|
||||
from .llms.llamafile.chat.transformation import (
|
||||
LlamafileChatConfig as _LlamafileChatConfig,
|
||||
)
|
||||
|
|
@ -2137,6 +2148,7 @@ if TYPE_CHECKING:
|
|||
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
|
||||
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
|
||||
DeepInfraConfig: Type[_DeepInfraConfig]
|
||||
ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
|
||||
LlamafileChatConfig: Type[_LlamafileChatConfig]
|
||||
LMStudioChatConfig: Type[_LMStudioChatConfig]
|
||||
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OllamaDecisionsConfig",
|
||||
"LayaDecisionsConfig",
|
||||
"BespokeDecisionsConfig",
|
||||
"EuJevDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
"AzureDecisionsConfig",
|
||||
"AzureAIDecisionsConfig",
|
||||
|
|
@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"MistralEmbeddingConfig",
|
||||
"OpenAIImageVariationConfig",
|
||||
"DeepInfraConfig",
|
||||
"ClfAiGatewayConfig",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
"TopazImageVariationConfig",
|
||||
"OpenAITextCompletionConfig",
|
||||
|
|
@ -753,6 +755,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
|
||||
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
|
||||
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
|
||||
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
|
||||
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
|
||||
|
|
@ -973,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OpenAIImageVariationConfig",
|
||||
),
|
||||
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
|
||||
"ClfAiGatewayConfig": (
|
||||
".llms.clf_ai_gateway.chat.transformation",
|
||||
"ClfAiGatewayConfig",
|
||||
),
|
||||
"DeepgramAudioTranscriptionConfig": (
|
||||
".llms.deepgram.audio_transcription.transformation",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
|
|
|
|||
|
|
@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
REDACTED,
|
||||
python_redact_string,
|
||||
python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
redact_string as redact_secret_string,
|
||||
|
|
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
|
|||
return redact_internal_details(value)
|
||||
|
||||
|
||||
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _redacted_client_field(key: str | None, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return redact_internal_details(redact_structured_value(key, value))
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
child_key: _redacted_client_field(child_key, child)
|
||||
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
|
||||
}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
|
||||
if not isinstance(value, (int, float)):
|
||||
return value
|
||||
text: Final = str(value)
|
||||
return value if redact_structured_value(key, text) == text else REDACTED
|
||||
|
||||
|
||||
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return fields
|
||||
return _redacted_client_field(None, dict(fields))
|
||||
|
||||
|
||||
def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
||||
"""Render a record's ``color_message`` against its args, or None if absent.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,9 @@ The following caching mechanisms are supported:
|
|||
2. **RedisSemanticCache**
|
||||
3. **QdrantSemanticCache**
|
||||
4. **InMemoryCache**
|
||||
5. **DiskCache**
|
||||
6. **S3Cache**
|
||||
7. **AzureBlobCache**
|
||||
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
5. **S3Cache**
|
||||
6. **AzureBlobCache**
|
||||
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
|
||||
## Folder Structure
|
||||
|
||||
|
|
@ -20,7 +19,6 @@ litellm/caching/
|
|||
├── base_cache.py
|
||||
├── caching.py
|
||||
├── caching_handler.py
|
||||
├── disk_cache.py
|
||||
├── dual_cache.py
|
||||
├── in_memory_cache.py
|
||||
├── qdrant_semantic_cache.py
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from .azure_blob_cache import AzureBlobCache
|
||||
from .caching import Cache, LiteLLMCacheType
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ def resolve_embedding_max_input_tokens(
|
|||
return configured_max_input_tokens
|
||||
if router is None:
|
||||
return None
|
||||
deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
|
||||
deployment_max_input_tokens, _ = router.registry().group(embedding_model, field="token_limits")
|
||||
return deployment_max_input_tokens
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from contextlib import contextmanager
|
|||
from enum import Enum
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_service_target, service_target
|
||||
|
|
@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
|||
|
||||
from .azure_blob_cache import AzureBlobCache
|
||||
from .base_cache import BaseCache
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
@ -44,6 +43,9 @@ from .s3_cache import S3Cache
|
|||
RESPONSE_CACHE_TARGET: Final = "llm_response"
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
|
||||
_DECODED_LITERAL: Final = TypeAdapter(object)
|
||||
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
|
||||
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -153,7 +155,7 @@ class Cache:
|
|||
Initializes the cache based on the given type.
|
||||
|
||||
Args:
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
|
||||
|
||||
# Redis Cache Args
|
||||
host (str, optional): The host address for the Redis cache. Required if type is "redis".
|
||||
|
|
@ -174,9 +176,6 @@ class Cache:
|
|||
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
|
||||
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
|
||||
|
||||
# Disk Cache Args
|
||||
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
|
||||
|
||||
# S3 Cache Args
|
||||
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
|
||||
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
|
||||
|
|
@ -211,7 +210,7 @@ class Cache:
|
|||
if not redis_startup_nodes:
|
||||
_env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
|
||||
if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
|
||||
redis_startup_nodes = json.loads(_env_cluster_nodes)
|
||||
redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
|
||||
|
||||
if redis_startup_nodes:
|
||||
# Only pass GCP parameters if they are provided
|
||||
|
|
@ -306,8 +305,11 @@ class Cache:
|
|||
account_url=azure_account_url,
|
||||
container=azure_blob_container,
|
||||
)
|
||||
elif type == LiteLLMCacheType.DISK:
|
||||
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
|
||||
else:
|
||||
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
|
||||
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
|
||||
if disk_cache_dir is not None:
|
||||
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
|
||||
if "cache" not in litellm.input_callback:
|
||||
litellm.input_callback.append("cache")
|
||||
if "cache" not in litellm.success_callback:
|
||||
|
|
@ -1044,7 +1046,7 @@ def enable_cache(
|
|||
Enable cache with the specified configuration.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
|
||||
host (Optional[str]): The host address of the cache server. Defaults to None.
|
||||
port (Optional[str]): The port number of the cache server. Defaults to None.
|
||||
password (Optional[str]): The password for the cache server. Defaults to None.
|
||||
|
|
@ -1056,7 +1058,7 @@ def enable_cache(
|
|||
None
|
||||
|
||||
Raises:
|
||||
None
|
||||
ValueError: If no cache is set yet and the cache type is unsupported.
|
||||
"""
|
||||
print_verbose("LiteLLM: Enabling Cache")
|
||||
if "cache" not in litellm.input_callback:
|
||||
|
|
@ -1091,7 +1093,7 @@ def update_cache(
|
|||
Update the cache for LiteLLM.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
|
||||
host (Optional[str]): The host of the cache. Defaults to None.
|
||||
port (Optional[str]): The port of the cache. Defaults to None.
|
||||
password (Optional[str]): The password for the cache. Defaults to None.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ This file is a wrapper around caching.py
|
|||
|
||||
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
|
||||
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
|
||||
|
||||
In each method it will call the appropriate method from caching.py
|
||||
"""
|
||||
|
|
@ -830,6 +830,13 @@ class LLMCachingHandler:
|
|||
if is_internal_call.get():
|
||||
return
|
||||
|
||||
if logging_obj.defer_async_logging and not logging_obj.stream:
|
||||
from litellm.utils import dispatch_success_logging
|
||||
|
||||
logging_obj.model_call_details["cache_hit"] = cache_hit # rebind-ok: capture cache status before deferring
|
||||
dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
|
||||
return
|
||||
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
try:
|
||||
import diskcache as dc
|
||||
except ModuleNotFoundError as e:
|
||||
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
|
||||
|
||||
# if users don't provider one, use the default litellm cache
|
||||
if disk_cache_dir is None:
|
||||
self.disk_cache = dc.Cache(".litellm_cache")
|
||||
else:
|
||||
self.disk_cache = dc.Cache(disk_cache_dir)
|
||||
|
||||
def set_cache(self, key, value, **kwargs):
|
||||
if "ttl" in kwargs:
|
||||
self.disk_cache.set(key, value, expire=kwargs["ttl"])
|
||||
else:
|
||||
self.disk_cache.set(key, value)
|
||||
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
self.set_cache(key=key, value=value, **kwargs)
|
||||
|
||||
async def async_set_cache_pipeline(self, cache_list, **kwargs):
|
||||
for cache_key, cache_value in cache_list:
|
||||
if "ttl" in kwargs:
|
||||
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
|
||||
else:
|
||||
self.set_cache(key=cache_key, value=cache_value)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
return None
|
||||
|
||||
def batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
def increment_cache(self, key, value: int, **kwargs) -> int:
|
||||
with self.disk_cache.transact():
|
||||
cached_value: Final = self.get_cache(key=key)
|
||||
init_value: Final = cached_value if isinstance(cached_value, int) else 0
|
||||
new_value: Final = init_value + value
|
||||
self.set_cache(key, new_value, **kwargs)
|
||||
return new_value
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
return self.get_cache(key=key, **kwargs)
|
||||
|
||||
async def async_batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
async def async_increment(self, key, value: int, **kwargs) -> int:
|
||||
return self.increment_cache(key=key, value=value, **kwargs)
|
||||
|
||||
def flush_cache(self):
|
||||
self.disk_cache.clear()
|
||||
|
||||
async def disconnect(self):
|
||||
pass
|
||||
|
||||
def delete_cache(self, key):
|
||||
self.disk_cache.pop(key)
|
||||
|
|
@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
|
|||
close them, and each bucket is ordered by deadline, so a reap walks the entries
|
||||
that are due rather than the whole queue.
|
||||
|
||||
A bucket whose loop has been collected is dropped, and its clients are left to the
|
||||
collector, as they are when the queue is full.
|
||||
|
||||
The queue holds its clients weakly, so waiting out a grace window never keeps
|
||||
alive anything the collector would have reclaimed first.
|
||||
"""
|
||||
|
|
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -44,6 +48,8 @@ from collections.abc import Awaitable, Callable, Iterator
|
|||
from dataclasses import dataclass, replace
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.constants import (
|
||||
EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
|
||||
EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
|
||||
|
|
@ -53,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
|
|||
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
|
||||
|
||||
_BucketKey = str | int
|
||||
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
|
||||
_LOOP_TOKEN_COUNTER: Final = itertools.count()
|
||||
_LOOP_TOKEN_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -63,12 +72,12 @@ class _PendingClose:
|
|||
collector would otherwise have reclaimed first.
|
||||
|
||||
``needs_loop`` is set for a client whose close is a coroutine; those can only
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_id``.
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_token``.
|
||||
A client that closes synchronously carries neither constraint.
|
||||
"""
|
||||
|
||||
client_ref: "weakref.ref[object]"
|
||||
loop_id: int | None
|
||||
loop_token: int | None
|
||||
needs_loop: bool
|
||||
close_after: float
|
||||
|
||||
|
|
@ -77,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
|
|||
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
|
||||
if not pending.needs_loop:
|
||||
return _CLOSABLE_ANYWHERE
|
||||
if pending.loop_id is None:
|
||||
if pending.loop_token is None:
|
||||
return _CLOSABLE_ON_ANY_LOOP
|
||||
return pending.loop_id
|
||||
return pending.loop_token
|
||||
|
||||
|
||||
def _running_loop_id() -> int | None:
|
||||
def running_loop_token() -> int | None:
|
||||
try:
|
||||
return id(asyncio.get_running_loop())
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
try:
|
||||
token: Final[int | None] = _LOOP_TOKENS.get(loop)
|
||||
if token is not None:
|
||||
return token
|
||||
with _LOOP_TOKEN_LOCK:
|
||||
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
|
||||
except TypeError:
|
||||
return id(loop)
|
||||
|
||||
|
||||
def _close_function(client: object) -> Callable[[], object] | None:
|
||||
|
|
@ -94,13 +111,24 @@ def _close_function(client: object) -> Callable[[], object] | None:
|
|||
return close_fn
|
||||
|
||||
|
||||
def _transport_of(client: object) -> object:
|
||||
"""The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
|
||||
def _mounted_transports(holder: object) -> tuple[object, ...]:
|
||||
mounts: Final[object] = getattr(holder, "_mounts", None)
|
||||
if not isinstance(mounts, dict):
|
||||
return ()
|
||||
return tuple(
|
||||
mounted
|
||||
for mounted in mounts.values() # pyright: ignore[reportUnknownVariableType] # untyped httpx mount map
|
||||
if isinstance(mounted, (httpx.AsyncBaseTransport, httpx.BaseTransport))
|
||||
)
|
||||
|
||||
|
||||
def _transports_of(client: object) -> tuple[object, ...]:
|
||||
"""The httpx transports behind an SDK wrapper, a litellm handler, or a bare client, proxy mounts included."""
|
||||
for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
|
||||
transport: object = getattr(holder, "_transport", None)
|
||||
if transport is not None:
|
||||
return transport
|
||||
return None
|
||||
return (transport, *_mounted_transports(holder))
|
||||
return ()
|
||||
|
||||
|
||||
def _connection_is_idle(connection: object) -> bool:
|
||||
|
|
@ -123,6 +151,17 @@ def _pool_has_busy_connection(transport: object) -> bool | None:
|
|||
)
|
||||
|
||||
|
||||
def _transport_in_flight(transport: object) -> bool:
|
||||
spread_in_flight: Final[object] = getattr(transport, "has_request_in_flight", None)
|
||||
if callable(spread_in_flight):
|
||||
return bool(spread_in_flight())
|
||||
pooled_busy: Final = _pool_has_busy_connection(transport)
|
||||
if pooled_busy is not None:
|
||||
return pooled_busy
|
||||
session: Final[object] = getattr(transport, "client", None)
|
||||
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
|
||||
|
||||
|
||||
def _has_connection_in_flight(client: object) -> bool:
|
||||
"""Whether the client is servicing a request right now.
|
||||
|
||||
|
|
@ -138,12 +177,7 @@ def _has_connection_in_flight(client: object) -> bool:
|
|||
try:
|
||||
if getattr(getattr(client, "connection_pool", None), "_in_use_connections", None):
|
||||
return True
|
||||
transport: Final = _transport_of(client)
|
||||
pooled_busy: Final = _pool_has_busy_connection(transport)
|
||||
if pooled_busy is not None:
|
||||
return pooled_busy
|
||||
session: Final[object] = getattr(transport, "client", None)
|
||||
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
|
||||
return any(_transport_in_flight(transport) for transport in _transports_of(client))
|
||||
except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle
|
||||
return False
|
||||
|
||||
|
|
@ -167,6 +201,8 @@ class EvictedClientCloser:
|
|||
self._clock = clock
|
||||
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
|
||||
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
|
||||
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
|
||||
self._pending_count = 0
|
||||
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
|
||||
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
|
||||
|
|
@ -197,13 +233,19 @@ class EvictedClientCloser:
|
|||
close_fn: Final = _close_function(client)
|
||||
if close_fn is None:
|
||||
return
|
||||
loop_token: Final = running_loop_token()
|
||||
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
self._discard_dead_loops()
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
if needs_loop and loop_token is not None:
|
||||
self._watch_loop(loop_token)
|
||||
self._enqueue(
|
||||
_PendingClose(
|
||||
client_ref=weakref.ref(client),
|
||||
loop_id=_running_loop_id(),
|
||||
needs_loop=inspect.iscoroutinefunction(close_fn),
|
||||
loop_token=loop_token,
|
||||
needs_loop=needs_loop,
|
||||
close_after=self._clock() + self._grace_seconds,
|
||||
)
|
||||
)
|
||||
|
|
@ -217,7 +259,7 @@ class EvictedClientCloser:
|
|||
if not self._pending_count:
|
||||
return
|
||||
now: Final = self._clock()
|
||||
for pending in self._take_due(_running_loop_id(), now):
|
||||
for pending in self._take_due(running_loop_token(), now):
|
||||
client = pending.client_ref()
|
||||
if client is None:
|
||||
continue
|
||||
|
|
@ -230,6 +272,29 @@ class EvictedClientCloser:
|
|||
def pending_count(self) -> int:
|
||||
return self._pending_count
|
||||
|
||||
def _watch_loop(self, loop_token: int) -> None:
|
||||
if loop_token in self._watched_loop_tokens:
|
||||
return
|
||||
try:
|
||||
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
|
||||
except TypeError:
|
||||
return
|
||||
self._watched_loop_tokens.add(loop_token)
|
||||
|
||||
def _discard_dead_loops(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
|
||||
def _discard_dead_loops_locked(self) -> None:
|
||||
while self._dead_loop_tokens:
|
||||
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
|
||||
|
||||
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
|
||||
self._watched_loop_tokens.discard(loop_token)
|
||||
bucket: Final = self._buckets.pop(loop_token, None)
|
||||
if bucket is not None:
|
||||
self._pending_count -= len(bucket)
|
||||
|
||||
def _enqueue(self, pending: _PendingClose) -> None:
|
||||
"""Append to the entry's bucket, dropping any dead entries it queues behind.
|
||||
|
||||
|
|
@ -238,6 +303,7 @@ class EvictedClientCloser:
|
|||
the front rather than having to be searched for.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
|
||||
while bucket and bucket[0].client_ref() is None:
|
||||
bucket.popleft()
|
||||
|
|
@ -245,9 +311,12 @@ class EvictedClientCloser:
|
|||
bucket.append(pending)
|
||||
self._pending_count += 1
|
||||
|
||||
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
|
||||
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (
|
||||
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
|
||||
)
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
|
||||
|
||||
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
|
|
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
|
|||
self.evicted_client_closer.schedule(evicted)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
def update_cache_key_with_event_loop(self, key):
|
||||
def update_cache_key_with_event_loop(self, key: str) -> str:
|
||||
"""
|
||||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
If none, use the key as is.
|
||||
"""
|
||||
try:
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
stringified_event_loop: Final = str(id(event_loop))
|
||||
return f"{key}-{stringified_event_loop}"
|
||||
except RuntimeError: # handle no current running event loop
|
||||
return key
|
||||
token: Final = running_loop_token()
|
||||
return f"{key}-{token}" if token is not None else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
def _scoped_key(self, key: str, loop_bound: bool) -> str:
|
||||
return self.update_cache_key_with_event_loop(key) if loop_bound else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
|
||||
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
|
||||
if litellm_owned_client:
|
||||
self.evicted_client_closer.mark_owned(value)
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
return super().set_cache(key, value, **kwargs)
|
||||
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
|
||||
|
||||
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
if litellm_owned_client:
|
||||
|
|
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
|
|||
key = self.update_cache_key_with_event_loop(key)
|
||||
return await super().async_set_cache(key, value, **kwargs)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
|
||||
scoped_key: Final = self._scoped_key(key, loop_bound)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
return super().get_cache(key, **kwargs)
|
||||
return super().get_cache(scoped_key, **kwargs)
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
async def async_get_cache(self, key: str, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
|
||||
return await super().async_get_cache(key, **kwargs)
|
||||
|
|
|
|||
|
|
@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
|
||||
self._structured_output: Final = structured_output
|
||||
self._first_message_item_done: bool = False
|
||||
self._reasoning_summary_streamed: bool = False
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: Union[str, "BaseModel"]
|
||||
|
|
@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
|
||||
replays_openai_history: bool = False,
|
||||
structured_output: bool = False,
|
||||
reasoning_summary_streamed: bool = False,
|
||||
) -> "ModelResponseStream":
|
||||
"""
|
||||
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
|
||||
|
|
@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
|
||||
replays_openai_history: Record the completed output's item sequence for the next turn
|
||||
structured_output: Only the first message item became content
|
||||
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
|
||||
|
||||
Returns:
|
||||
ModelResponseStream: OpenAI-formatted streaming chunk
|
||||
|
|
@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
else:
|
||||
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
|
||||
elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
|
||||
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
|
||||
elif event_type == "response.reasoning_summary_text.delta":
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=cast(int, parsed_chunk.get("summary_index")),
|
||||
index=0,
|
||||
delta=Delta(reasoning_content=content_part),
|
||||
)
|
||||
]
|
||||
|
|
@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map=self._tool_call_index_map,
|
||||
replays_openai_history=self._replays_openai_history,
|
||||
structured_output=self._structured_output,
|
||||
reasoning_summary_streamed=self._reasoning_summary_streamed,
|
||||
)
|
||||
)
|
||||
if self._structured_output and _is_message_item_done(chunk):
|
||||
self._first_message_item_done = True
|
||||
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
|
||||
self._reasoning_summary_streamed = True
|
||||
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
|
||||
|
||||
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -753,7 +753,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
|
|||
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
|
||||
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
|
||||
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
|
||||
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
|
||||
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
|
||||
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
|
||||
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
|
||||
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
||||
"low": 1,
|
||||
|
|
@ -813,6 +819,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"ollama",
|
||||
"ollama_chat",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"mistral",
|
||||
"groq",
|
||||
|
|
@ -1012,6 +1019,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"api.perplexity.ai",
|
||||
"api.endpoints.anyscale.com/v1",
|
||||
"api.deepinfra.com/v1/openai",
|
||||
"api.clfaigateway.dev/v1",
|
||||
"api.mistral.ai/v1",
|
||||
"codestral.mistral.ai/v1/chat/completions",
|
||||
"codestral.mistral.ai/v1/fim/completions",
|
||||
|
|
@ -1083,6 +1091,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"deepseek",
|
||||
"tencent",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"xinference",
|
||||
"xai",
|
||||
|
|
|
|||
|
|
@ -153,7 +153,9 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
cached_cost_map_key,
|
||||
cached_get_model_info_helper,
|
||||
mapped_model_fill_missing,
|
||||
model_info_with_pricing_overrides,
|
||||
runtime_model_cost_registrations,
|
||||
token_counter,
|
||||
|
|
@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
|
|||
"""
|
||||
1. If custom pricing is true, return received model name
|
||||
2. If base_model is set (e.g. for azure models), return that
|
||||
3. If completion response has model set return that
|
||||
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
|
||||
4. Check if model is passed in return that
|
||||
"""
|
||||
|
||||
return_model: str | None = None
|
||||
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
completion_response_model: str | None = None
|
||||
if completion_response is not None:
|
||||
if isinstance(completion_response, BaseModel):
|
||||
completion_response_model = getattr(completion_response, "model", None)
|
||||
elif isinstance(completion_response, dict):
|
||||
completion_response_model = completion_response.get("model", None)
|
||||
completion_response_model: Final = _get_response_model(completion_response)
|
||||
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
|
||||
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
|
||||
requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
configured_model: Final = model or (
|
||||
completion_response_model if isinstance(completion_response_model, str) else None
|
||||
|
|
@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
|
|||
else:
|
||||
return_model = model
|
||||
|
||||
elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
|
||||
elif base_model is not None or bedrock_pricing_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model
|
||||
|
||||
elif provider_response_model is not None:
|
||||
return _served_or_priced_requested_model(
|
||||
provider_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
elif completion_response_model is None and hidden_params is not None:
|
||||
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
|
||||
return_model = hidden_params.get("model", model)
|
||||
|
||||
if return_model is None and completion_response_model is not None:
|
||||
return_model = completion_response_model
|
||||
return _served_or_priced_requested_model(
|
||||
completion_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
if return_model is None and model is not None:
|
||||
return_model = model
|
||||
|
||||
if (
|
||||
return_model is not None
|
||||
and custom_llm_provider is not None
|
||||
and not _model_contains_known_llm_provider(return_model)
|
||||
): # add provider prefix if not already present, to match model_cost
|
||||
provider_prefix: Final = (
|
||||
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
|
||||
)
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
|
||||
return return_model
|
||||
if return_model is None:
|
||||
return None
|
||||
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
|
||||
|
||||
|
||||
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
|
||||
|
||||
|
||||
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
|
||||
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
|
||||
return model
|
||||
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
|
||||
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
|
||||
|
||||
|
||||
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
|
||||
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
|
||||
|
||||
|
||||
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
key: Final = cached_cost_map_key(model, custom_llm_provider)
|
||||
if key is None:
|
||||
return False
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
if entry is None:
|
||||
return _rule_declares_rates(model, custom_llm_provider)
|
||||
fill_missing: Final = mapped_model_fill_missing(key, entry)
|
||||
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
|
||||
|
||||
|
||||
def _served_or_priced_requested_model(
|
||||
served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
|
||||
) -> str:
|
||||
served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
|
||||
if requested_model is None:
|
||||
return served
|
||||
requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
|
||||
if served in (requested, requested.partition("/")[2]):
|
||||
return served
|
||||
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
|
||||
if _resolves_to_rates(served, pricing_provider):
|
||||
return served
|
||||
return requested if _resolves_to_rates(requested, pricing_provider) else served
|
||||
|
||||
|
||||
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
|
||||
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
|
||||
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
|
||||
|
|
@ -1887,7 +1922,10 @@ def completion_cost(
|
|||
video_resolution=video_resolution,
|
||||
)
|
||||
elif call_type in _SPEECH_CALL_TYPES:
|
||||
prompt_characters = litellm.utils.count_characters(text=prompt)
|
||||
prompt_characters = _speech_billed_characters(
|
||||
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
|
||||
prompt,
|
||||
)
|
||||
elif call_type in _TRANSCRIPTION_CALL_TYPES:
|
||||
# Check _hidden_params first (duration stored there to
|
||||
# avoid polluting the response body), then fall back to
|
||||
|
|
@ -2366,6 +2404,11 @@ def response_cost_calculator(
|
|||
raise e
|
||||
|
||||
|
||||
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
|
||||
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
|
||||
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
|
||||
|
||||
|
||||
def _video_input_image_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -3508,7 +3551,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
|
|||
lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
|
||||
session that should bill nothing fell through to the public rates instead.
|
||||
"""
|
||||
resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
entries: Final = (
|
||||
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
|
||||
litellm.model_cost.get(model_name),
|
||||
|
|
@ -3704,9 +3747,9 @@ def handle_realtime_transcription_cost_calculation(
|
|||
return 0.0
|
||||
|
||||
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
|
||||
model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
override_info: Final = (
|
||||
_get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
)
|
||||
|
||||
total_cost = 0.0
|
||||
|
|
@ -3737,13 +3780,6 @@ def _get_transcription_model_name_from_results(
|
|||
return None
|
||||
|
||||
|
||||
def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
|
||||
try:
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ from litellm.main import (
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.llms.openai import ImageGenerationRequestQuality
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
LITELLM_IMAGE_VARIATION_PROVIDERS,
|
||||
LlmProviders,
|
||||
|
|
@ -1010,6 +1010,7 @@ def image_edit(
|
|||
optional_params=dict(image_edit_request_params),
|
||||
litellm_params={
|
||||
**image_edit_request_params,
|
||||
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": model_info,
|
||||
"vertex_location": litellm_params.vertex_location,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class ImageEditRequestUtils:
|
|||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
drop_params: bool | None = None,
|
||||
additional_drop_params: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Get optional parameters for the image edit API.
|
||||
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
except Exception:
|
||||
return 0
|
||||
|
||||
async def send_daily_reports(self, router) -> bool:
|
||||
async def send_daily_reports(self, router: Router) -> bool:
|
||||
"""
|
||||
Send a daily report on:
|
||||
- Top 5 deployments with most failed requests
|
||||
|
|
@ -365,7 +365,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
False -> if not sent
|
||||
"""
|
||||
|
||||
ids: Final = router.get_model_ids()
|
||||
ids: Final = router.registry().names(kind="ids")
|
||||
|
||||
# get keys
|
||||
failed_request_keys: Final = [f"{id}:{SlackAlertingCacheKeys.failed_requests_key.value}" for id in ids]
|
||||
|
|
@ -430,7 +430,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
message += "\tNone\n"
|
||||
for i in range(len(top_5_failed)):
|
||||
key = failed_request_keys[top_5_failed[i]].split(":")[0]
|
||||
_deployment = router.get_model_info(key)
|
||||
_deployment = router.registry().deployment_dict(key)
|
||||
if isinstance(_deployment, dict):
|
||||
deployment_name = _deployment["litellm_params"].get("model", "")
|
||||
else:
|
||||
|
|
@ -452,7 +452,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
message += "\tNone\n"
|
||||
for i in range(len(top_5_slowest)):
|
||||
key = latency_keys[top_5_slowest[i]].split(":")[0]
|
||||
_deployment = router.get_model_info(key)
|
||||
_deployment = router.registry().deployment_dict(key)
|
||||
if _deployment is not None:
|
||||
deployment_name = _deployment["litellm_params"].get("model", "")
|
||||
else:
|
||||
|
|
@ -802,7 +802,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
if self.llm_router is None:
|
||||
return
|
||||
|
||||
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = self.llm_router.registry().deployment(deployment_id)
|
||||
|
||||
if deployment is None:
|
||||
return
|
||||
|
|
@ -953,7 +953,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
return
|
||||
|
||||
### EXTRACT MODEL DETAILS ###
|
||||
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = self.llm_router.registry().deployment(deployment_id)
|
||||
if deployment is None:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
|
|||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
client: Final = get_httpx_client()
|
||||
try:
|
||||
response: Final = client.post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
response: Final = get_httpx_client().post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
finally:
|
||||
client.close()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
|
|
|||
|
|
@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
|
|||
import urllib.parse
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
class _PromptVersionResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
|
||||
|
||||
data: dict[str, object] | None = None
|
||||
|
||||
|
||||
def _sanitize_id(identifier: str) -> str:
|
||||
"""Reject path traversal characters and URL-encode the identifier."""
|
||||
if any(c in identifier for c in ("/", "\\", "#", "?")):
|
||||
|
|
@ -71,8 +79,7 @@ class ArizePhoenixClient:
|
|||
response: Final = self.http_handler.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
data: Final = response.json()
|
||||
return data.get("data")
|
||||
return _PromptVersionResponse.model_validate(response.json()).data
|
||||
|
||||
except Exception as e:
|
||||
# Check if it's an HTTP error
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
|
||||
every call but one whose logging payload shows it never reached a provider."""
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
|
||||
if not _a_destination_carries_the_trace():
|
||||
return False
|
||||
call: Final = LLMCallEvent.from_dict(kwargs)
|
||||
return call.payload is None or call.reached_a_provider
|
||||
|
||||
|
||||
def _a_destination_carries_the_trace() -> bool:
|
||||
"""Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
|
||||
trace."""
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
|
||||
return ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
|
||||
|
||||
class LangfuseOtelLogger(OpenTelemetry):
|
||||
def __init__(self, config=None, *args, **kwargs):
|
||||
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
|
||||
|
|
@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"""
|
||||
return None
|
||||
|
||||
def _emit_once(
|
||||
self,
|
||||
kwargs: dict[str, object], # mutable-ok: the v1 dedupe marker is written into the request's kwargs
|
||||
*scope: object,
|
||||
) -> bool:
|
||||
"""Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
|
||||
return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope) # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict[str, object], # mutable-ok: the v1 hook contract
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: str | None = None,
|
||||
) -> None:
|
||||
"""Record the proxy failure of a request no Langfuse destination carries."""
|
||||
if not _a_destination_carries_the_trace():
|
||||
await super().async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
request_data, original_exception, user_api_key_dict, traceback_str
|
||||
)
|
||||
|
||||
async def async_service_success_hook(self, *args, **kwargs):
|
||||
"""
|
||||
Langfuse should not receive service success logs.
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
Tracer = _Tracer
|
||||
Context = _Context
|
||||
SpanExporter = _SpanExporter
|
||||
|
|
@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span = None
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_span is guaranteed to be not None here
|
||||
if hasattr(parent_span, "set_status"):
|
||||
if parent_span is not None and hasattr(parent_span, "set_status"):
|
||||
parent_span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
# Raw-request as direct child of parent_span
|
||||
|
|
@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# record error on parent span (keeps hierarchy shallow)
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_otel_span is guaranteed to be not None here
|
||||
if parent_otel_span.is_recording():
|
||||
if parent_otel_span is not None and parent_otel_span.is_recording():
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
|
|||
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
|
||||
from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
|
|
@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
|
|||
mcp_tool_call_span_name,
|
||||
service_span_name,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
|
||||
|
|
@ -168,6 +169,9 @@ class SpanEmitter:
|
|||
self._mappers: list[AttributeMapper] = (
|
||||
list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
|
||||
)
|
||||
self._langfuse: Final = (
|
||||
None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
|
||||
)
|
||||
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
|
||||
# only — the value is unused — so it behaves like a capped set.
|
||||
self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
|
||||
|
|
@ -295,17 +299,27 @@ class SpanEmitter:
|
|||
)
|
||||
else None
|
||||
)
|
||||
mapped: Final = MappingProxyType(
|
||||
{key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
for_langfuse: Final = (
|
||||
dict(
|
||||
sorted(
|
||||
self._langfuse.map(data).items(),
|
||||
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
)
|
||||
)
|
||||
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
else _NO_ATTRIBUTES
|
||||
)
|
||||
mapped: Final = MappingProxyType({**configured, **for_langfuse})
|
||||
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
|
||||
reserved: Final = len(stamped_later.keys() - mapped.keys())
|
||||
overwrites: Final = carried_keys(span) & mapped.keys()
|
||||
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
|
||||
# The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
|
||||
# so a span at its attribute limit loses nothing else to them.
|
||||
fitted: Final = fit_indexed_messages(
|
||||
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
|
||||
attribute_budget(span, reserved),
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
|
||||
)
|
||||
for key, value in mapped.items():
|
||||
if key in overwrites or key in fitted:
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
tracer_provider: TracerProvider | None = None,
|
||||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
console_default: bool = False,
|
||||
**kwargs: Unpack[_CustomLoggerOptions],
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._tracer_provider: TracerProvider = (
|
||||
tracer_provider
|
||||
if tracer_provider is not None
|
||||
else build_tracer_provider(self.config, tenant_overrides=True)
|
||||
else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
|
||||
)
|
||||
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
|
||||
self._metrics_recorder = self._init_metrics(meter_provider)
|
||||
|
|
@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
mappers=resolve_mappers(self.config.mapper_names),
|
||||
event_recorder=self._init_events(logger_provider),
|
||||
)
|
||||
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
|
||||
self._tenant_tracers = TenantTracerCache(
|
||||
self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
|
||||
)
|
||||
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
|
||||
self._init_otel_logger_on_litellm_proxy()
|
||||
|
||||
|
|
@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
|
||||
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
|
||||
# upstream stamp, since the payload proves the call completed.
|
||||
if carrier is None and (
|
||||
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
|
||||
):
|
||||
if carrier is None and not call.reached_a_provider:
|
||||
return None
|
||||
try:
|
||||
return self._finish_carrier(carrier, call, start_time, end_time)
|
||||
|
|
@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
|
|||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
settings: Mapping[str, object] = MappingProxyType({}),
|
||||
console_default: bool = False,
|
||||
) -> OpenTelemetryV2:
|
||||
return _logger_class(config)(
|
||||
config=config,
|
||||
|
|
@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
|
|||
tracer_provider=tracer_provider,
|
||||
logger_provider=logger_provider,
|
||||
meter_provider=meter_provider,
|
||||
console_default=console_default,
|
||||
**settings,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,18 @@ class LangfuseMapper:
|
|||
),
|
||||
}
|
||||
|
||||
#: Every attribute this mapper can write on a model call.
|
||||
LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
*_LLM_CALL_ATTRS,
|
||||
*_BLOB_ATTRS,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LANGFUSE_TRACE_USER_ID,
|
||||
LANGFUSE_TRACE_SESSION_ID,
|
||||
LANGFUSE_TRACE_TAGS,
|
||||
}
|
||||
)
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
|
|
|
|||
|
|
@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
|
|||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec ``credential_gated_exporters`` appends, which would
|
||||
otherwise eat itself when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
||||
|
||||
class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
|
||||
def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
|
||||
super().__init__(settings_cls)
|
||||
|
|
|
|||
|
|
@ -272,6 +272,11 @@ class LLMCallEvent:
|
|||
trace: TraceControls
|
||||
session_id: str | None
|
||||
|
||||
@property
|
||||
def reached_a_provider(self) -> bool:
|
||||
"""Whether the call was handed to a provider or answered from the response cache."""
|
||||
return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
|
|||
LANGFUSE_OBSERVATION_OUTPUT,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LangfuseMapper,
|
||||
)
|
||||
from litellm.integrations.otel.model.config import (
|
||||
CaptureMessageContent,
|
||||
|
|
@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
|
|||
OpenTelemetryV2Config,
|
||||
capture_message_content_from_env,
|
||||
captures_span_content,
|
||||
is_unconfigured_placeholder,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
|
|
@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
|
|||
return scope is not None and scope.name == LITELLM_TRACER_NAME
|
||||
|
||||
|
||||
def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
|
||||
"""``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
|
||||
def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
|
||||
"""``span`` without the Langfuse attributes only a Langfuse destination asked for.
|
||||
|
||||
Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
|
||||
That is the request metadata the Langfuse mapper writes and, on a model call written by
|
||||
a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
|
||||
every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
|
||||
destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
|
||||
its own code set.
|
||||
"""
|
||||
attributes: Final = span.attributes or _NO_ATTRIBUTES
|
||||
if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
|
||||
dropped: Final = (
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
if langfuse_mapped or not is_llm_call_span(span)
|
||||
else LangfuseMapper.LLM_CALL_ATTRIBUTES
|
||||
)
|
||||
if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
|
||||
return span
|
||||
return _CountedSpanView(
|
||||
span,
|
||||
MappingProxyType(
|
||||
{key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
|
||||
),
|
||||
span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -658,6 +666,7 @@ def _for_destination(
|
|||
span: ReadableSpan,
|
||||
destination: "OtelDestination",
|
||||
default_capture: str,
|
||||
langfuse_mapped: bool,
|
||||
) -> ReadableSpan:
|
||||
"""The view of ``span`` a tenant destination receives.
|
||||
|
||||
|
|
@ -672,13 +681,16 @@ def _for_destination(
|
|||
and the query string of the request URL, which can hold the same key. A
|
||||
destination whose effective setting disables span content loses the prompt and
|
||||
response content too, and a destination for any backend but Langfuse loses the
|
||||
request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
|
||||
stays, so the tenant still gets the whole trace tree.
|
||||
Langfuse attributes only a Langfuse destination asked for (see
|
||||
:func:`without_destination_langfuse_attributes`). The span itself stays, so the
|
||||
tenant still gets the whole trace tree.
|
||||
"""
|
||||
extra: Final = destination.resource_attributes
|
||||
defaults: Final = destination.resource_defaults
|
||||
source: Final = (
|
||||
span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
|
||||
span
|
||||
if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
|
||||
else without_destination_langfuse_attributes(span, langfuse_mapped)
|
||||
)
|
||||
attributes: Final = source.attributes or _NO_ATTRIBUTES
|
||||
database: Final = _is_database_span(attributes)
|
||||
|
|
@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
|
||||
pending_drains: int = _MAX_PENDING_DRAINS,
|
||||
drain_pool: _DrainPool | None = None,
|
||||
langfuse_mapped: bool = True,
|
||||
) -> None:
|
||||
self._operator_sinks: Final = operator_sinks
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
self._excluded_db_systems: Final = excluded_db_systems
|
||||
self._default_capture: Final = default_capture
|
||||
self._default_capture_by_backend: Final = default_capture_by_backend
|
||||
|
|
@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
try:
|
||||
processor.on_end(
|
||||
_scoped(
|
||||
_for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
|
||||
_for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
|
||||
destination.span_scope,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span
|
||||
|
|
@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
|
|||
return self._inner.force_flush(timeout_millis)
|
||||
|
||||
|
||||
class _LangfuseRequestMetadataFilter(SpanProcessor):
|
||||
"""Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
|
||||
class _DestinationLangfuseAttributeFilter(SpanProcessor):
|
||||
"""Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
|
||||
exports them."""
|
||||
|
||||
def __init__(self, inner: SpanProcessor) -> None:
|
||||
def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
|
||||
self._inner: Final = inner
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
|
||||
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
|
||||
self._inner.on_start(span, parent_context)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._inner.on_end(without_langfuse_request_metadata(span))
|
||||
self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._inner.shutdown()
|
||||
|
|
@ -1491,13 +1508,18 @@ def _spec_processor(
|
|||
spec: ExporterSpec,
|
||||
use_simple_processor: bool | None,
|
||||
strip_message_content: bool,
|
||||
strip_langfuse_request_metadata: bool,
|
||||
strip_destination_langfuse_attributes: bool,
|
||||
langfuse_mapped: bool,
|
||||
) -> SpanProcessor:
|
||||
exporting: Final = _processor_for(
|
||||
_exporter_from_spec(spec),
|
||||
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
|
||||
)
|
||||
scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
|
||||
scoped: Final = (
|
||||
_DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
|
||||
if strip_destination_langfuse_attributes
|
||||
else exporting
|
||||
)
|
||||
return _MessageContentFilter(scoped) if strip_message_content else scoped
|
||||
|
||||
|
||||
|
|
@ -1530,6 +1552,7 @@ def build_tracer_provider(
|
|||
use_simple_processor: bool | None = None,
|
||||
tenant_overrides: bool = False,
|
||||
strip_message_content: bool = False,
|
||||
console_default: bool = False,
|
||||
) -> TracerProvider:
|
||||
"""Build the shared :class:`TracerProvider`.
|
||||
|
||||
|
|
@ -1548,11 +1571,16 @@ def build_tracer_provider(
|
|||
|
||||
``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
|
||||
alone; a collector or any other backend in the same config keeps the full tree.
|
||||
In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
|
||||
goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
|
||||
Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
|
||||
Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
|
||||
which any logger writes for a request with a Langfuse destination.
|
||||
``strip_message_content`` removes prompt and response content before every
|
||||
exporter, for a routed provider whose spans can carry content another
|
||||
destination asked for.
|
||||
|
||||
The console spec ``config._normalize`` folds in when nothing was configured
|
||||
prints only under ``console_default``, which the ``otel`` callback passes: a
|
||||
vendor preset or the proxy's background global logger was never asked to print.
|
||||
"""
|
||||
provider: Final = TracerProvider(resource=build_resource(config))
|
||||
provider.sampler = _RequestExclusionSampler(provider.sampler)
|
||||
|
|
@ -1564,19 +1592,20 @@ def build_tracer_provider(
|
|||
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
|
||||
return provider
|
||||
|
||||
langfuse_mapped: Final = _maps_langfuse(config)
|
||||
# ``config._normalize`` guarantees at least one spec (it folds the top-level
|
||||
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
|
||||
writes_langfuse: Final = "langfuse" in config.mapper_names or any(
|
||||
spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
|
||||
)
|
||||
for spec in config.exporters:
|
||||
if spec.requires_headers and not spec.headers:
|
||||
continue
|
||||
if is_unconfigured_placeholder(spec) and not console_default:
|
||||
continue
|
||||
processor = _spec_processor(
|
||||
spec,
|
||||
use_simple_processor,
|
||||
strip_message_content,
|
||||
strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
langfuse_mapped=langfuse_mapped,
|
||||
)
|
||||
owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
|
||||
scope = _operator_scope(config, spec)
|
||||
|
|
@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
|
|||
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
|
||||
config per v2 logger since each keeps its own provider and still writes its
|
||||
account, so an additive destination pointing at any of them is delivered once
|
||||
rather than twice. ``excluded_db_systems`` only filters what the fan-out
|
||||
delivers, never the operator's own exporters.
|
||||
rather than twice. The first is the config of the logger that owns ``provider``,
|
||||
whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
|
||||
the fan-out delivers, never the operator's own exporters.
|
||||
"""
|
||||
with _FAN_OUT_ATTACH_LOCK:
|
||||
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
|
||||
|
|
@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
|
|||
excluded_db_systems=excluded_db_systems,
|
||||
default_capture=_operator_capture_mode(configs),
|
||||
default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
|
||||
langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
|
||||
"""Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
|
||||
return "langfuse" in config.mapper_names
|
||||
|
||||
|
||||
def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
|
||||
"""The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
|
||||
return next(
|
||||
|
|
|
|||
|
|
@ -163,10 +163,12 @@ class TenantTracerCache:
|
|||
config: OpenTelemetryV2Config,
|
||||
callback_name: str | None,
|
||||
tracer_name: str,
|
||||
console_default: bool = False,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._callback_name = callback_name
|
||||
self._tracer_name = tracer_name
|
||||
self._console_default = console_default
|
||||
# Guards the three mutable structures below: ``pre_call`` can run on
|
||||
# thread-pool workers concurrently with the event loop, so cache
|
||||
# updates, span counts, and retirement must be atomic.
|
||||
|
|
@ -284,6 +286,7 @@ class TenantTracerCache:
|
|||
built: Final = build_tracer_provider(
|
||||
self._routed_config(credential_headers, project_headers, endpoint, service_name),
|
||||
strip_message_content=not self._config.capture_span_content,
|
||||
console_default=self._console_default,
|
||||
)
|
||||
self._providers[cache_key] = built
|
||||
return built
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
|
|||
_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _arize_project() -> Mapping[str, str]:
|
||||
def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
from litellm.integrations.arize.arize import ArizeLogger
|
||||
|
||||
project: Final = ArizeLogger.get_arize_config().project_name
|
||||
|
|
@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
|
|||
return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
|
||||
|
||||
|
||||
#: Resource attributes the operator chose for a backend, applied to every export the
|
||||
#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
|
||||
{"arize": _arize_project}
|
||||
def _stands_in_for_the_legacy_langfuse_logger() -> bool:
|
||||
"""Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
|
||||
of its own sends for a key's or team's ``langfuse_otel``."""
|
||||
return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
|
||||
|
||||
|
||||
def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
"""The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
|
||||
logger wrote it."""
|
||||
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
|
||||
|
||||
if not _stands_in_for_the_legacy_langfuse_logger():
|
||||
return _NO_ATTRS
|
||||
environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
|
||||
|
||||
|
||||
#: Resource attributes a backend gets on every export the team receives, over the
|
||||
#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
|
||||
MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
|
||||
)
|
||||
|
||||
#: Resource attributes a backend needs on every export to accept it, filled only where
|
||||
|
|
@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
|
|||
)
|
||||
|
||||
|
||||
def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
|
||||
def _resource_attributes(
|
||||
callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
|
||||
) -> Mapping[str, str]:
|
||||
backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**({"service.name": service_name} if service_name else {}),
|
||||
**(backend() if backend is not None else {}),
|
||||
**(backend(params) if backend is not None else {}),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -214,7 +233,7 @@ def destination_for(
|
|||
return OtelDestination(
|
||||
endpoint=endpoint,
|
||||
headers=MappingProxyType(dict(headers)),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name, params),
|
||||
resource_defaults=_resource_defaults(callback_name),
|
||||
callback_name=callback_name,
|
||||
protocol=protocol,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
|
||||
|
||||
|
||||
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
|
||||
|
|
@ -35,14 +35,3 @@ def credential_gated_exporters(
|
|||
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
|
||||
ExporterSpec(owner=owner, requires_headers=True),
|
||||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec this module appends, which would otherwise eat itself
|
||||
when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
|
|
|||
|
|
@ -468,15 +468,16 @@ def _bounded_requested_model_label(requested_model: object, router_originated: b
|
|||
llm_router: Final = _get_proxy_llm_router()
|
||||
if llm_router is None:
|
||||
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
if llm_router.is_recognized_model(requested_model):
|
||||
registry: Final = llm_router.registry()
|
||||
if registry.resolve(requested_model, kind="recognized"):
|
||||
return requested_model
|
||||
if requested_model in llm_router.team_public_model_names:
|
||||
if requested_model in registry.names(kind="team_public"):
|
||||
return requested_model
|
||||
if llm_router.pattern_router.route(requested_model) is not None:
|
||||
if registry.patterns().route(requested_model) is not None:
|
||||
return requested_model
|
||||
if any(
|
||||
team_pattern_router.route(requested_model) is not None
|
||||
for team_pattern_router in llm_router.team_pattern_routers.values()
|
||||
for team_pattern_router in registry.patterns(team=True).values()
|
||||
):
|
||||
return requested_model
|
||||
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
|
|
@ -3410,7 +3411,7 @@ class PrometheusLogger(CustomLogger):
|
|||
OpenAI / Anthropic / Azure return remaining tokens/requests in response
|
||||
headers, but Bedrock and Vertex AI do not. This fallback computes
|
||||
``configured_limit - current_usage`` via
|
||||
``Router.get_remaining_model_group_usage`` so the same gauges are
|
||||
``Router.health().usage(model_group, kind="remaining")`` so the same gauges are
|
||||
emitted for every provider when tpm/rpm is configured on the
|
||||
deployment.
|
||||
"""
|
||||
|
|
@ -3437,7 +3438,7 @@ class PrometheusLogger(CustomLogger):
|
|||
return
|
||||
|
||||
try:
|
||||
remaining_usage: Final = await llm_router.get_remaining_model_group_usage(model_group)
|
||||
remaining_usage: Final = await llm_router.health().usage(model_group, kind="remaining")
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"Prometheus: get_remaining_model_group_usage failed for model_group=%s: %s",
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ from litellm.litellm_core_utils.llm_judge import (
|
|||
)
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
|
||||
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
|
||||
|
|
@ -790,9 +789,7 @@ class ActiveShadowEvalJob(LiteLLMBaseModel):
|
|||
def _canonical_group(router: "Router | None", model_group: str) -> str:
|
||||
"""A model group in the one spelling both a job's scope and a request's model compare
|
||||
under: an alias resolves to its target so the two never fail to match on spelling."""
|
||||
return (
|
||||
resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
|
||||
) or model_group
|
||||
return (router.registry().resolve(model_group, kind="alias") if router is not None else None) or model_group
|
||||
|
||||
|
||||
def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -132,7 +133,7 @@ class _SearchToolLitellmParams(TypedDict, total=False):
|
|||
|
||||
|
||||
class _SearchToolConfig(TypedDict, total=False):
|
||||
search_tool_name: str
|
||||
search_tool_name: ReadOnly[str]
|
||||
litellm_params: ReadOnly[_SearchToolLitellmParams | None]
|
||||
|
||||
|
||||
|
|
@ -1768,10 +1769,13 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
return None
|
||||
|
||||
def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None":
|
||||
if llm_router is None or not hasattr(llm_router, "search_tools"):
|
||||
def _select_search_tool_from_router(self, llm_router: "Router | None") -> "_SearchToolConfig | None":
|
||||
if llm_router is None:
|
||||
return None
|
||||
search_tools: Final = tuple(getattr(llm_router, "search_tools", None) or ())
|
||||
settings: Final = llm_router.settings()
|
||||
if not settings.is_set("search_tools"):
|
||||
return None
|
||||
search_tools: Final = tuple(settings.search_tools or ())
|
||||
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
|
||||
|
||||
def _select_search_tool_from_list(
|
||||
|
|
|
|||
|
|
@ -292,6 +292,10 @@ def get_llm_provider(
|
|||
elif endpoint == "api.deepinfra.com/v1/openai":
|
||||
custom_llm_provider = "deepinfra"
|
||||
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
|
||||
elif endpoint == "api.clfaigateway.dev/v1":
|
||||
custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch
|
||||
clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
|
||||
dynamic_api_key = api_key or clf_env_key
|
||||
elif endpoint == "api.mistral.ai/v1":
|
||||
custom_llm_provider = "mistral"
|
||||
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
|
||||
|
|
@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
|
|||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "clf_ai_gateway":
|
||||
(
|
||||
api_base, # rebind-ok: resolved base/key are what this function returns, like every branch
|
||||
dynamic_api_key, # rebind-ok: see above
|
||||
) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "empower":
|
||||
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# What is this?
|
||||
## Common Utility file for Logging handler
|
||||
# Logging function -> log the exact model details + what's being sent | Non-Blocking
|
||||
import asyncio
|
||||
import copy
|
||||
import datetime
|
||||
import functools
|
||||
|
|
@ -96,6 +97,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
MODEL_ACCESS_GROUP_METADATA_KEY,
|
||||
is_unbilled_non_inference_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost, PreparedCost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
|
||||
bedrock_guardrail_trace,
|
||||
cost_breakdown_with_guardrail,
|
||||
|
|
@ -200,6 +202,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
batch_chat_usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose
|
||||
|
|
@ -813,6 +816,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
# Initialize cost breakdown field
|
||||
self.cost_breakdown: CostBreakdown | None = None
|
||||
self.finalized_cost: FinalizedCost | None = None
|
||||
self.prepared_cost: PreparedCost | None = None
|
||||
self.cost_allowance_discount: float | None = None
|
||||
self.cost_finalization_lock = asyncio.Lock()
|
||||
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
|
||||
self.billed_token_rates: BilledTokenRates | None = None
|
||||
self._provider_reported_cost: float | None = None
|
||||
|
||||
|
|
@ -874,6 +882,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"current_attempt_api_call_start_time": None,
|
||||
"litellm_params": litellm_params,
|
||||
}
|
||||
new_call.finalized_cost = None
|
||||
new_call.prepared_cost = None
|
||||
new_call.cost_allowance_discount = None
|
||||
new_call.cost_finalization_lock = asyncio.Lock()
|
||||
new_call.deferred_success_result = None
|
||||
new_call.streaming_chunks = []
|
||||
new_call.sync_streaming_chunks = []
|
||||
return new_call
|
||||
|
|
@ -2079,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
if isinstance(priced_result, LiteLLMBatch) and (
|
||||
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
|
||||
):
|
||||
return 0.0
|
||||
|
||||
# Fallback: extract router_model_id from litellm_params when not available
|
||||
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
|
||||
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
|
||||
|
|
@ -2462,6 +2480,55 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
|
||||
if self.prepared_cost is not None:
|
||||
return self.prepared_cost
|
||||
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
|
||||
deferred: Final = self.deferred_success_result
|
||||
prepared: Final = self.prepare_success_logging(
|
||||
result=deferred[0] if deferred is not None else response,
|
||||
start_time=deferred[1] if deferred is not None else self.start_time,
|
||||
end_time=deferred[2] if deferred is not None else now.replace(tzinfo=None),
|
||||
cache_hit=recorded_cache_hit if isinstance(recorded_cache_hit, bool) else None,
|
||||
standard_logging_object=None,
|
||||
build_logging_payload=True,
|
||||
publish_logging_payload=False,
|
||||
)
|
||||
payload: Final = prepared[3]
|
||||
raw_cost: Final[object] = self.model_call_details.get("response_cost")
|
||||
if payload is None or not isinstance(raw_cost, (int, float)):
|
||||
return None
|
||||
recorded_usage: Final[object] = self.model_call_details.get("combined_usage_object")
|
||||
response_usage: Final[object] = getattr(prepared[2], "usage", None)
|
||||
normalized_usage: Final = (
|
||||
recorded_usage
|
||||
if isinstance(recorded_usage, Usage)
|
||||
else response_usage
|
||||
if isinstance(response_usage, Usage)
|
||||
else None
|
||||
)
|
||||
usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(
|
||||
_extract_response_obj_and_hidden_params(prepared[2], None)[0],
|
||||
combined_usage_object=normalized_usage,
|
||||
).model_copy(deep=True)
|
||||
self.prepared_cost = PreparedCost(
|
||||
raw_cost,
|
||||
{
|
||||
**payload,
|
||||
"response": None,
|
||||
"messages": None,
|
||||
"model_map_information": copy.deepcopy(payload["model_map_information"]),
|
||||
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
|
||||
},
|
||||
usage,
|
||||
)
|
||||
self.record_partial_usage_for_failure(usage, raw_cost)
|
||||
return self.prepared_cost
|
||||
|
||||
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
|
||||
self.finalized_cost = finalized
|
||||
self.model_call_details["response_cost"] = finalized.amount
|
||||
|
||||
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
|
||||
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
|
||||
self.model_call_details["combined_usage_object"] = usage
|
||||
|
|
@ -2812,12 +2879,16 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
logging_result, "_hidden_params", {}
|
||||
)
|
||||
|
||||
if self.model_call_details.get("cache_hit") is True:
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
elif self.model_call_details.get("cache_hit") is True:
|
||||
self.model_call_details["response_cost"] = 0.0
|
||||
elif "response_cost" in hidden_params:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
|
||||
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
):
|
||||
# Preserve response_cost if already calculated (e.g., by pass-through
|
||||
# handlers like Gemini/Vertex which call completion_cost directly).
|
||||
# Do not preserve 0 from failure_handler on intermediate router retries.
|
||||
|
|
@ -2936,7 +3007,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
standard_logging_object: StandardLoggingPayload | None = None,
|
||||
build_logging_payload: bool = True,
|
||||
) -> tuple[datetime.datetime, datetime.datetime, object]:
|
||||
prepared: Final = self._prepare_success_logging(
|
||||
prepared: Final = self.prepare_success_logging(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
@ -2947,7 +3018,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
return prepared[0], prepared[1], prepared[2]
|
||||
|
||||
def _prepare_success_logging(
|
||||
def prepare_success_logging(
|
||||
self,
|
||||
result: object,
|
||||
start_time: datetime.datetime | None,
|
||||
|
|
@ -2968,7 +3039,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
self.model_call_details["cache_hit"] = cache_hit
|
||||
self.model_call_details["cache_hit"] = (
|
||||
self.prepared_cost.payload["cache_hit"]
|
||||
if cache_hit is None and self.prepared_cost is not None
|
||||
else cache_hit
|
||||
)
|
||||
|
||||
if self.call_type == CallTypes.anthropic_messages.value:
|
||||
result = self._anthropic_messages_logged_response(result=result)
|
||||
|
|
@ -3007,6 +3082,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
if (
|
||||
litellm.max_budget
|
||||
and publish_logging_payload
|
||||
and self.stream is False
|
||||
and result is not None
|
||||
and isinstance(result, dict)
|
||||
|
|
@ -3037,10 +3113,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
publish_logging_payload: bool,
|
||||
) -> StandardLoggingPayload | None:
|
||||
if standard_logging_object is not None:
|
||||
effective_payload: Final = (
|
||||
self.finalized_cost.logging_payload(standard_logging_object)
|
||||
if self.finalized_cost is not None
|
||||
else standard_logging_object
|
||||
)
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
if not publish_logging_payload:
|
||||
return standard_logging_object.copy()
|
||||
self.model_call_details["standard_logging_object"] = standard_logging_object
|
||||
return standard_logging_object
|
||||
return effective_payload.copy()
|
||||
self.model_call_details["standard_logging_object"] = effective_payload
|
||||
return effective_payload
|
||||
if result is None or self.stream is True:
|
||||
# Streaming reaches here before its cost is known, so the cost
|
||||
# is seeded to None, but only when nothing has already
|
||||
|
|
@ -3269,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
finally:
|
||||
self._restore_correlation_context()
|
||||
|
||||
def _is_batch_poll_without_logging(self, result: object) -> bool:
|
||||
litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
|
||||
return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
|
||||
def _success_handler_body(
|
||||
self,
|
||||
result: object = None,
|
||||
|
|
@ -3281,7 +3368,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
|
||||
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
|
||||
return
|
||||
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
|
||||
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
result=result,
|
||||
|
|
@ -3783,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## CALCULATE COST FOR BATCH JOBS
|
||||
if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
|
||||
litellm_params = self.litellm_params or {}
|
||||
litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
|
||||
if (
|
||||
litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
): # polling job will query these frequently, don't spam db logs
|
||||
if self._is_batch_poll_without_logging(result):
|
||||
return
|
||||
|
||||
batch_cost: Final = kwargs.get("batch_cost", None)
|
||||
|
|
@ -4204,7 +4287,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# chunks already delivered; the router stashes that recovered usage as
|
||||
# ``combined_usage_object`` and pre-computes its cost, so preserve it
|
||||
# here instead of zeroing the spend on an otherwise-failed request.
|
||||
if self.model_call_details.get("combined_usage_object") is None:
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
elif self.model_call_details.get("combined_usage_object") is None:
|
||||
self.model_call_details["response_cost"] = 0
|
||||
|
||||
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
|
||||
|
|
@ -5544,7 +5629,7 @@ def build_custom_logger_compatible_class(
|
|||
return callback
|
||||
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
|
||||
otel_logger_v2: Final = build_otel_v2_logger(
|
||||
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
|
||||
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
|
||||
)
|
||||
_in_memory_loggers.append(otel_logger_v2)
|
||||
_maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
|
||||
|
|
@ -7367,6 +7452,12 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
|
|||
)
|
||||
|
||||
|
||||
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
|
||||
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
|
||||
finalized: Final = getattr(logging_obj, "finalized_cost", None)
|
||||
return finalized if isinstance(finalized, FinalizedCost) else None
|
||||
|
||||
|
||||
def get_standard_logging_object_payload(
|
||||
kwargs: dict | None,
|
||||
init_response_obj: object,
|
||||
|
|
@ -7396,7 +7487,8 @@ def get_standard_logging_object_payload(
|
|||
# Extract usage as a plain dict, avoiding Pydantic round-trip
|
||||
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
|
||||
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
|
||||
)
|
||||
usage_dict: Final = (
|
||||
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
|
||||
|
|
@ -7470,7 +7562,10 @@ def get_standard_logging_object_payload(
|
|||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
response_cost: Final[float] = llm_response_cost + guardrail_cost
|
||||
finalized_cost: Final = _finalized_cost_of(logging_obj)
|
||||
response_cost: Final[float] = (
|
||||
finalized_cost.amount if finalized_cost is not None else llm_response_cost + guardrail_cost
|
||||
)
|
||||
|
||||
# clean up litellm hidden params
|
||||
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
|
||||
|
|
@ -7537,13 +7632,17 @@ def get_standard_logging_object_payload(
|
|||
):
|
||||
model_name = response_model_name
|
||||
|
||||
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
|
||||
(
|
||||
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
|
||||
if uncached_price is not None
|
||||
else logging_obj.cost_breakdown
|
||||
),
|
||||
guardrail_cost,
|
||||
request_cost_breakdown: Final = (
|
||||
finalized_cost.logging_breakdown()
|
||||
if finalized_cost is not None
|
||||
else cost_breakdown_with_guardrail(
|
||||
(
|
||||
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
|
||||
if uncached_price is not None
|
||||
else logging_obj.cost_breakdown
|
||||
),
|
||||
guardrail_cost,
|
||||
)
|
||||
)
|
||||
captured_baseline: Final = logging_obj.baseline_observation
|
||||
autorouter_savings: Final = (
|
||||
|
|
@ -7600,6 +7699,7 @@ def get_standard_logging_object_payload(
|
|||
model=model_name,
|
||||
metadata=clean_metadata,
|
||||
cache_key=clean_hidden_params["cache_key"],
|
||||
cost_finalized=finalized_cost is not None,
|
||||
response_cost=response_cost,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
autorouter_savings=autorouter_savings,
|
||||
|
|
|
|||
61
litellm/litellm_core_utils/llm_cost_calc/finalization.py
Normal file
61
litellm/litellm_core_utils/llm_cost_calc/finalization.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import CostBreakdown, StandardLoggingPayload, Usage
|
||||
|
||||
BreakdownValue: TypeAlias = float | str | None | Mapping[str, float]
|
||||
_BREAKDOWN_FIELDS: Final = TypeAdapter(Mapping[str, BreakdownValue])
|
||||
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PreparedCost:
|
||||
amount: float
|
||||
payload: StandardLoggingPayload
|
||||
usage: Usage
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FinalizedCost:
|
||||
amount: float
|
||||
breakdown: Mapping[str, BreakdownValue] | None
|
||||
allowance_discount: float
|
||||
|
||||
def logging_payload(self, payload: StandardLoggingPayload) -> StandardLoggingPayload:
|
||||
return {
|
||||
**payload,
|
||||
"response_cost": self.amount,
|
||||
"cost_breakdown": self.logging_breakdown(),
|
||||
"cost_finalized": True,
|
||||
}
|
||||
|
||||
def logging_breakdown(self) -> CostBreakdown | None:
|
||||
return _COST_BREAKDOWN.validate_python(self.breakdown) if self.breakdown is not None else None
|
||||
|
||||
|
||||
def finalize_cost(amount: float, breakdown: CostBreakdown | None, allowance_discount: float = 0.0) -> FinalizedCost:
|
||||
discount: Final = min(max(amount, 0.0), max(allowance_discount, 0.0))
|
||||
total: Final = amount - discount
|
||||
existing: Final = _BREAKDOWN_FIELDS.validate_python(breakdown) if breakdown is not None else None
|
||||
adjusted: Final = (
|
||||
{
|
||||
**(existing or {}),
|
||||
"total_cost": total,
|
||||
"original_cost": (breakdown or {}).get("original_cost", amount),
|
||||
"discount_amount": (breakdown or {}).get("discount_amount", 0.0) + discount,
|
||||
}
|
||||
if discount > 0
|
||||
else existing
|
||||
)
|
||||
frozen: Final = (
|
||||
MappingProxyType(
|
||||
{key: MappingProxyType(value) if isinstance(value, Mapping) else value for key, value in adjusted.items()}
|
||||
)
|
||||
if adjusted is not None
|
||||
else None
|
||||
)
|
||||
return FinalizedCost(amount=total, breakdown=frozen, allowance_discount=discount)
|
||||
|
|
@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
|
|||
"output_cost_per_image_token",
|
||||
)
|
||||
|
||||
|
||||
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
|
||||
return pricing is not None and (
|
||||
"input_cost_per_pixel" in pricing
|
||||
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
|
||||
)
|
||||
|
||||
|
||||
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
|
||||
# request in the cost-calc path, so the f-strings are built once here instead
|
||||
# of being rebuilt for every model_info key on every call. Longest-first so a
|
||||
|
|
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
|
||||
|
||||
|
||||
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
|
||||
|
|
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
|
|||
|
||||
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
|
||||
value: Final = _requested_image_param(optional_params, "size")
|
||||
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
|
||||
|
||||
def get_web_search_requests(server_tool_use: object) -> int | None:
|
||||
|
|
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
|
|||
|
||||
|
||||
def _model_priced_image_cost_calculator(
|
||||
custom_llm_provider: str | None, model: str
|
||||
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
|
||||
) -> _ModelPricedImageCostCalculator | None:
|
||||
match custom_llm_provider:
|
||||
case litellm.LlmProviders.RECRAFT.value:
|
||||
|
|
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
|
|||
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
|
||||
|
||||
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
|
||||
case litellm.LlmProviders.DASHSCOPE.value:
|
||||
if _deployment_prices_images_without_an_output_price(pricing):
|
||||
return None
|
||||
from litellm.llms.dashscope.image_generation.cost_calculator import (
|
||||
cost_calculator as dashscope_image_cost_calculator,
|
||||
)
|
||||
|
||||
return dashscope_image_cost_calculator
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
|
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
|
|||
)
|
||||
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
|
||||
pricing: Final = deployment_pricing(model_info)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
|
||||
|
||||
if model_priced_calculator is not None:
|
||||
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ def judge_target(router: Router | None, model: str, team_id: str | None = None)
|
|||
only for its own team and a team's own deployment resolves for nobody else, so asking
|
||||
without it answers for a caller who does not exist.
|
||||
"""
|
||||
served: Final = router.resolved_litellm_models(model, team_id=team_id) if router is not None else ()
|
||||
served: Final = router.registry().names(model, kind="litellm_models", team_id=team_id) if router is not None else ()
|
||||
if served:
|
||||
return JudgeTarget("router", frozenset(_provider_qualified(m) or m for m in served))
|
||||
qualified: Final = _provider_qualified(model)
|
||||
|
|
@ -148,7 +148,7 @@ def attached_auth(request_data: object) -> UserAPIKeyAuth | None:
|
|||
|
||||
|
||||
def _named_route(router: Router, deployment_id: str) -> bool:
|
||||
deployment: Final = router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = router.registry().deployment(deployment_id)
|
||||
if deployment is None:
|
||||
return False
|
||||
public_names: Final = (deployment.model_name, deployment.model_info.team_public_model_name or "")
|
||||
|
|
@ -167,7 +167,7 @@ def caller_team_for(router: Router | None, model: str, auth: UserAPIKeyAuth | No
|
|||
team_id: Final = auth.team_id if auth is not None else None
|
||||
if team_id is None or router is None or judge_target(router, model).via != "nothing":
|
||||
return None
|
||||
team_route: Final = router.get_candidate_model_ids_for_route(model, team_id)
|
||||
team_route: Final = router.registry().names(model, kind="route_candidates", team_id=team_id)
|
||||
return team_id if team_route and all(_named_route(router, deployment_id) for deployment_id in team_route) else None
|
||||
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue