scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

chore: merge main into litellm_remove_lit002_dict_ban
Some checks failed
LiteLLM Rust / rust-lint (push) Waiting to run
Details
LiteLLM Rust / rust-test (push) Waiting to run
Details
LiteLLM Rust / rust-wheel (push) Waiting to run
Details
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Details
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Details
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details

Browse source
This commit is contained in:
mateo 2026-10-08 00:31:06 +00:00
commit 0be8fd5ec8
396 changed files with 10866 additions and 21862 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.circleci/config.yml
View file

@ -3145,10 +3145,10 @@ jobs:
name: Test provider capture and replay harness
command: |
mkdir -p test-results/provider-replay-harness
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
--junitxml=test-results/provider-replay-harness/junit.xml \
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
tests/e2e_harness/test_provider_edge.py tests/e2e_harness/test_fixture_bundle.py \
tests/e2e_harness/test_fixture_canonical.py tests/e2e_harness/test_fixture_mode.py \
tests/code_coverage_tests/test_provider_replay_harness.py \
tests/code_coverage_tests/test_provider_cache.py
- store_test_results:

4
.circleci/scripts/classify_changes.sh
View file

@ -24,8 +24,8 @@ while IFS= read -r file || [ -n "$file" ]; do
has_mcp_dependencies=true ;;
esac
case "$file" in
tests/e2e/*/*.py) : ;;
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
tests/e2e/*/*.py | tests/e2e_harness/*/*.py) : ;;
tests/e2e/*.py | tests/e2e_harness/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
has_provider_harness=true ;;
esac
case "$file" in

8
.circleci/scripts/run_integration.sh
View file

@ -193,10 +193,10 @@ fi
if [ "$suite" = providers ]; then
INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --tb=short --noconftest -o addopts= \
--strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
tests/e2e/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
tests/e2e/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
tests/e2e_harness/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
tests/e2e_harness/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
--junitxml="$results/replay-controls.xml"
fi

4
.github/CODEOWNERS vendored
View file

@ -1,2 +1,2 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerrylu-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerrylu-berri

28
.github/workflows/image-scan.yml vendored
View file

@ -18,6 +18,7 @@ on:
- backend/Dockerfile
- backend/main.py
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/e2e/migrations/lens_compose_smoke.sh
- docker/component_entrypoint.sh
@ -52,7 +53,7 @@ jobs:
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 15
timeout-minutes: 45
permissions:
contents: read
strategy:
@ -79,30 +80,7 @@ jobs:
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c '
import os
import lens.worker
from lens.release import release_tag
from lens.trace_store import trace_store
assert os.getuid() == 65532
assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"]
with trace_store() as store:
assert store.count() == 0
'
- name: Reject a dependency whose hash has changed
run: |
docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps .
sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \
deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock"
if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \
--entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \
--require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \
> "$RUNNER_TEMP/hash-check.log" 2>&1; then
echo "::error::Dependency hash mismatch was accepted"
exit 1
fi
cat "$RUNNER_TEMP/hash-check.log"
grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log"
lens-worker-scan --version | grep -F "litellm-lens $RELEASE_TAG protocol="
- name: Download Grype v0.114.0
env:
ARCH: ${{ matrix.arch }}

141
.github/workflows/lens-worker.yml vendored
View file

@ -5,6 +5,7 @@ on:
branches: [main, litellm_oss_branch, "litellm_**"]
paths:
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
@ -12,6 +13,7 @@ on:
branches: [main]
paths:
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
@ -29,15 +31,35 @@ jobs:
permissions:
contents: read
packages: write
id-token: write
runs-on: ubuntu-latest
timeout-minutes: 10
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: ubuntu-latest
- arch: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build Lens worker
run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
- name: Build native Lens service
env:
RELEASE_TAG: sha-${{ github.sha }}
run: docker build --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-worker .
- name: Verify version and unprivileged runtime
env:
RELEASE_TAG: sha-${{ github.sha }}
run: bash deploy/lens/smoke.sh lens-worker "$RELEASE_TAG"
- name: Verify confined Python on the native architecture
env:
RELEASE_TAG: sha-${{ github.sha }}
run: |
docker build --target smoke --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-smoke .
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges lens-smoke
- name: Reject custom builds without a matching release tag
run: |
if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
@ -45,86 +67,45 @@ jobs:
exit 1
fi
grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
- name: Verify standalone imports with a read-only filesystem
run: |
docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
--security-opt no-new-privileges --entrypoint python \
lens-worker:${{ github.sha }} -c '
import os
import lens.worker
from lens.trace_store import trace_store
assert os.getuid() == 65532
with trace_store() as store:
assert store.count() == 0
'
- name: Prepare test-only coverage tool
run: |
coverage_directory=$(mktemp -d "$RUNNER_TEMP/lens-coverage.XXXXXX")
curl --fail --silent --show-error --location \
https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl \
--output "$coverage_directory/coverage.whl"
printf '%s %s\n' 8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1 \
"$coverage_directory/coverage.whl" | sha256sum --check
chmod 777 "$coverage_directory"
echo "LENS_COVERAGE_DIRECTORY=$coverage_directory" >> "$GITHUB_ENV"
- name: Verify confined Python execution
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_python_smoke.py:/app/python_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.python /app/python_smoke.py
- name: Verify workspace investigation and live review output
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_context_smoke.py:/app/context_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.context /app/context_smoke.py
- name: Verify default workspace recovery after Python scratch storage fills
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.storage /app/storage_smoke.py
- name: Map native worker coverage to repository sources
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--security-opt no-new-privileges -w /workspace \
-v "$PWD/litellm/proxy/lens:/workspace/litellm/proxy/lens:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint /bin/sh lens-worker:${{ github.sha }} \
-c 'python -m coverage combine && python -m coverage xml'
- name: Upload native worker coverage
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: ${{ env.LENS_COVERAGE_DIRECTORY }}/lens-worker.xml
root_dir: ${{ github.workspace }}
flags: lens-worker
fail_ci_if_error: false
- name: Publish versioned Lens worker
- name: Publish development architecture
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
env:
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}-${{ matrix.arch }}
ARCH: ${{ matrix.arch }}
run: |
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
docker tag lens-worker "$IMAGE"
docker push "$IMAGE"
mkdir -p digests
docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE" > "digests/$ARCH"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
with:
name: lens-digest-${{ matrix.arch }}
path: digests/
retention-days: 1
publish:
name: Publish Lens development index
needs: lens-worker-image
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-digest-*
merge-multiple: true
path: digests
- name: Publish both tested architectures
env:
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}
run: |
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
docker tag lens-worker:${{ github.sha }} "$IMAGE"
docker push "$IMAGE"
docker buildx imagetools create --tag "$IMAGE" "$(cat digests/amd64)" "$(cat digests/arm64)"
printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

14
.github/workflows/test-linting.yml vendored
View file

@ -174,23 +174,25 @@ jobs:
- name: Check tests/e2e basedpyright (zero errors)
if: steps.changes.outputs.decision != 'skip'
run: |
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' ':(glob)tests/e2e_harness/**/*.py' pyrightconfig.json | grep -q .; then
uv run --no-sync basedpyright tests/e2e tests/e2e_harness
else
echo "No changed tests/e2e Python files; skipping."
fi
- name: Run the claude_code harness unit tests
- name: Run the e2e harness tests
if: steps.changes.outputs.decision != 'skip'
env:
LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
run: |
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/claude_code/**/*.py' ':(glob)tests/e2e/*.py' tests/e2e/claude_code/cron_vm/install_claude_code.sh pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
echo "No changed claude_code harness files; skipping."
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
echo "No changed e2e harness files; skipping."
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures tests/e2e/claude_code/_*_unit_tests
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
- name: Check for circular imports
if: steps.changes.outputs.decision != 'skip'

8
.github/workflows/test-rust.yml vendored
View file

@ -6,6 +6,8 @@ on:
- "litellm-rust/**"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/generate_lens_contract.py"
- "litellm/proxy/lens/**"
- "scripts/trace_codegen/**"
- "tests/test_litellm_rust/**"
- "litellm/integrations/custom_logger.py"
@ -35,6 +37,8 @@ on:
- "litellm-rust/**"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/generate_lens_contract.py"
- "litellm/proxy/lens/**"
- "scripts/trace_codegen/**"
- "tests/test_litellm_rust/**"
- "litellm/integrations/custom_logger.py"
@ -132,6 +136,10 @@ jobs:
working-directory: .
run: uv run scripts/generate_trace_types.py --check
- name: Check generated Lens contracts
working-directory: .
run: uv run scripts/generate_lens_contract.py --check
- run: cargo nextest run --workspace --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema
- run: cargo test --workspace --doc --locked

1
.github/workflows/test-unit.yml vendored
View file

@ -483,6 +483,7 @@ jobs:
tests/unit/sandbox
tests/unit/skills/test_skills_main.py
tests/unit/tracing
tests/proxy_behavior/lens/test_connection.py
workers: 2
reruns: 0
timeout-minutes: 20

4
Makefile
View file

@ -31,7 +31,7 @@ help:
@echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@ -211,7 +211,7 @@ lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_check_gate.py --base "$(BASE_REF)"
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
$(UV_RUN) basedpyright tests/e2e tests/e2e_harness
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.

51
deploy/lens/Dockerfile
View file

@ -1,36 +1,41 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13 build-base libseccomp-dev
ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy
WORKDIR /app
COPY deploy/lens/requirements.lock /tmp/requirements.lock
RUN uv venv --python python3.13 /app/.venv && \
uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock
RUN apk add --no-cache rust build-base cmake perl pkgconf openssl-dev libseccomp-dev python-3.13
WORKDIR /src
COPY .cargo/ .cargo/
COPY litellm-rust/ litellm-rust/
COPY litellm/proxy/lens/prompts/ litellm/proxy/lens/prompts/
WORKDIR /src/litellm-rust
ENV CARGO_PROFILE_RELEASE_DEBUG=0 CARGO_PROFILE_RELEASE_STRIP=symbols
RUN cargo build --locked --release -p litellm-lens
COPY deploy/lens/python_policy.c /tmp/python_policy.c
RUN cc -std=c11 -D_GNU_SOURCE -O2 -Wall -Wextra -Werror /tmp/python_policy.c -lseccomp -o /tmp/python-policy && \
/tmp/python-policy /app/python.seccomp
/tmp/python-policy /tmp/python.seccomp
FROM $LITELLM_RUNTIME_IMAGE AS runtime
FROM builder AS test-builder
RUN cargo test --locked --release -p litellm-lens --test sandbox --no-run --message-format=json > /tmp/test-artifacts.json && \
python3.13 -c 'import json, pathlib, shutil; rows = [json.loads(line) for line in pathlib.Path("/tmp/test-artifacts.json").read_text().splitlines()]; artifact, = [r["executable"] for r in rows if r.get("executable") and r["target"]["name"] == "sandbox"]; shutil.copyfile(artifact, "/tmp/lens-sandbox-tests")' && \
chmod 755 /tmp/lens-sandbox-tests
FROM $LITELLM_RUNTIME_IMAGE AS service
ARG LITELLM_RELEASE_TAG=""
RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
RUN apk add --no-cache python-3.13 setpriv
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \
PATH="/app/.venv/bin:${PATH}" \
PYTHONDONTWRITEBYTECODE=1
RUN apk add --no-cache python-3.13 setpriv libgcc libstdc++ openssl ca-certificates
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} PYTHONDONTWRITEBYTECODE=1
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
COPY litellm/proxy/lens/context_pipeline.py litellm/proxy/lens/agent_review.py litellm/proxy/lens/agent_runtime.py litellm/proxy/lens/agent_workspace.py litellm/proxy/lens/python_tool.py litellm/proxy/lens/activity.py litellm/proxy/lens/agent_context.py /app/lens/
COPY litellm/proxy/lens/reviews.py litellm/proxy/lens/reconciliation.py /app/lens/
COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
COPY --from=builder /app/python.seccomp /app/lens/python.seccomp
COPY --from=builder /src/litellm-rust/target/release/litellm-lens /usr/local/bin/litellm-lens
COPY --from=builder /tmp/python.seccomp /app/lens/python.seccomp
COPY deploy/lens/python_runtime.py /tmp/python_runtime.py
RUN python3.13 -S /tmp/python_runtime.py /app/lens/python-runtime.json && rm /tmp/python_runtime.py
USER 65532:65532
CMD ["python", "-m", "lens.worker"]
EXPOSE 4318
ENTRYPOINT ["/usr/local/bin/litellm-lens"]
FROM service AS smoke
COPY --from=test-builder /tmp/lens-sandbox-tests /usr/local/bin/lens-sandbox-tests
ENTRYPOINT ["/usr/local/bin/lens-sandbox-tests"]
CMD ["--ignored", "--nocapture", "--test-threads=1"]
FROM service AS runtime

7
deploy/lens/Dockerfile.dockerignore
View file

@ -1,12 +1,15 @@
**
!deploy/
!deploy/lens/
!deploy/lens/requirements.lock
!deploy/lens/python_policy.c
!deploy/lens/python_runtime.py
!litellm/
!litellm/proxy/
!litellm/proxy/lens/
!litellm/proxy/lens/*.py
!litellm/proxy/lens/prompts/
!litellm/proxy/lens/prompts/**
!.cargo/
!.cargo/**
!litellm-rust/
!litellm-rust/**
litellm-rust/target/

173
deploy/lens/README.md
View file

@ -1,112 +1,118 @@
# Lens worker
# Lens service
Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
Lens records agent activity and investigates it in a separate Rust service. LiteLLM serves model requests, the dashboard, and investigation settings. Lens owns trace ingestion and ClickHouse access; PostgreSQL stays with LiteLLM
## Install
Agent exporters send traces directly to Lens. LiteLLM sends its optional request logs through a bounded background queue. If Lens or ClickHouse is unavailable, model requests continue; traces can be delayed or dropped according to the exporter's retry policy. The gateway never waits for ClickHouse during startup or inference
Build LiteLLM and its worker from the same source commit with the same release identity. The worker runs separately and connects to your gateway using a limited worker token
## New local installation
### New local installation
Install Docker with Compose and Git. This builds LiteLLM and its worker from the same checkout and starts the existing local tracing stack:
Install Docker with Compose and Git, then build the gateway and Lens from one checkout:
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)"
export LENS_WORKER_IMAGE="litellm-lens-worker:${LITELLM_RELEASE_TAG}"
export OPENAI_API_KEY='sk-...'
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 24)"
export LITELLM_LENS_SERVICE_TOKEN="$(openssl rand -hex 32)"
export OPENAI_API_KEY='<your-provider-key>'
docker compose -f docker/docker-compose.tracing.yml up -d --build
```
Open `http://localhost:4002/ui/` and sign in as `admin` with the key saved in `.lens-dev/master_key`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
Save the generated keys privately and reuse them when restarting or upgrading. This stack binds to localhost and uses development database passwords; use your normal secrets, TLS, backups, and ingress for a hosted deployment
```bash
export LITELLM_URL=http://litellm:4000
export LENS_WORKER_TOKEN='<paste-your-worker-token>'
docker compose -f docker/docker-compose.tracing.yml -f deploy/lens/compose.yaml up -d
```
Open `http://localhost:4002/ui/` and sign in as `admin` with `LITELLM_MASTER_KEY`. Under **Lens > Traces > Set up tracing**, generate a tracing key and copy the ingestion URL. Local exporters use `http://localhost:4318`. Model calls keep their existing LiteLLM URL and model key
The worker joins the gateway's Docker network, and the dashboard shows **Worker connected**. Save the token privately for restarts and upgrades
Under **Lens > Investigations > Connect worker**, choose an analysis model and monthly budget. The deployed service connects automatically after you save these settings. There is no worker command or second token to copy
This stack is for local evaluation: it binds to localhost and uses development database credentials. For a hosted deployment, keep your normal database, keys, networking, and deployment process. Build both images from one source revision with the same `LITELLM_RELEASE_TAG`, publish the worker to your registry, and set `LENS_WORKER_IMAGE` on LiteLLM to that image
## Existing LiteLLM installation
### Existing LiteLLM installation
Keep your gateway, PostgreSQL database, deployment tool, and existing encryption keys. Deploy the matching Lens image, give it access to ClickHouse, and configure the service connection on LiteLLM
Keep your deployment and PostgreSQL database. A working gateway/worker pair can stay as it is until you upgrade both. For a gateway built from source, use its exact commit and `LITELLM_RELEASE_TAG`; a release version or the latest commit on `main` is not a substitute for that source identity
| Variable | LiteLLM | Lens service |
| --- | --- | --- |
| `LITELLM_LENS_SERVICE_TOKEN` | Same private random secret, at least 32 characters | Same secret |
| `LITELLM_LENS_URL` | Internal Lens URL, such as `http://lens-worker:4318` | Not needed |
| `LITELLM_LENS_PUBLIC_URL` | Ingestion base URL reachable by your agents | Not needed |
| `LITELLM_URL` | Not needed | LiteLLM URL reachable from Lens |
| `CLICKHOUSE_URL` | Remove it from Lens tracing configuration | ClickHouse HTTP URL with credentials |
| `CLICKHOUSE_DATABASE` | Not needed for Lens | Existing database name, defaults to `litellm` |
| `AGENT_TRACING_RETENTION_DAYS` | Not needed for Lens | Retention for traces and Lens request logs, defaults to `14` |
The public development package is `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`. It publishes amd64 images on Lens-related changes, so an arbitrary source commit may have no image. Check the exact image exists before using it. If it is unavailable, your gateway uses a different release identity, or you need native arm64, build the worker from the gateway's checkout:
Remove the old `general_settings.tracing.store` configuration used for Lens from LiteLLM. Keep unrelated logging integrations and their configuration. Only Lens should reach its ClickHouse database. The shared service secret is an infrastructure credential: keep it out of browser code, agent exporters, screenshots, and public ingress headers
Expose the Lens HTTP listener on port 4318 through TLS. Route `/lens-ingest` on your existing hostname directly to Lens at the load balancer, then set `LITELLM_LENS_PUBLIC_URL=https://<your-host>/lens-ingest`. The gateway must not proxy these uploads. Alternatively use a separate hostname and forward `/v1/` to Lens. Keep `/internal/` private; it requires the service secret
### Standalone Docker or a container host
Build from the same source commit and `LITELLM_RELEASE_TAG` as your running gateway:
```bash
export LITELLM_RELEASE_TAG='<gateway-release-identity>'
export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<your-image-tag>'
export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<image-tag>'
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
```
For a remote worker host, publish that image to a registry the host can pull from. Set the gateway's `LENS_WORKER_IMAGE` to the resulting image reference, restart the gateway using its normal deployment process, then copy its install command. Prefer the published image digest for hosted installations. Do not change the gateway's release identity just to accept another worker
Publish that image to a registry your host can pull from. Prefer a digest reference for hosted deployments. Public development images use `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`; check that the exact image exists before selecting it. An arbitrary commit may not have a published image
For Kubernetes or Render, run the standalone worker using `LITELLM_URL` and `LENS_WORKER_TOKEN` from setup. Keep existing databases and secrets. The worker needs no inbound port.
The image supports native amd64 and arm64. For worker-only Compose, use `deploy/lens/compose.yaml` with a private environment file containing `LENS_WORKER_IMAGE`, `LITELLM_URL`, `LITELLM_LENS_SERVICE_TOKEN`, and `CLICKHOUSE_URL`:
## Helm
```bash
docker compose --env-file /path/to/private/lens.env \
-f deploy/lens/compose.yaml up -d
```
The componentized source chart at `helm/litellm` includes an optional Lens worker. Use the chart from the same checkout as your gateway and keep your component image overrides in your values. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values:
The Compose listener binds to localhost. Your reverse proxy must reach it. On Render, run Lens as a web service with the same environment and listener port 4318, not an outbound-only background worker. Use `/health/live` for process health and `/health/ready` to check storage and tracing credentials
Lens does not need provider credentials, PostgreSQL credentials, a GPU, or the LiteLLM Python package. The image includes a small CPython runtime only for the investigator's confined calculation tool. Keep the shipped security settings, temporary filesystem, and resource limits
### Kubernetes with Helm
Both `helm/litellm` and `helm/litellm-helm` support the Lens service. Keep your existing release, namespace, values, and database configuration. Create two Secrets through your normal secret manager: `litellm-lens-service` with key `service-token`, and `litellm-lens-clickhouse` with key `url`
```yaml
lensWorker:
enabled: true
image:
repository: <your-worker-image-repository>
repository: <matching-worker-image-repository>
digest: sha256:<matching-worker-image-digest>
tokenSecret:
name: litellm-lens-worker
key: token
serviceTokenSecret:
name: litellm-lens-service
key: service-token
clickhouseSecret:
name: litellm-lens-clickhouse
key: url
clickhouseDatabase: litellm
retentionDays: 14
publicUrl: https://<your-litellm-host>/lens-ingest
```
Set the worker repository and digest explicitly to an image built from the gateway's source commit and release identity. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart and update the gateway and worker image overrides together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too
Set `clickhouseDatabase` and `retentionDays` to your existing database and retention before upgrading
## Standalone worker
When the chart's main ingress is enabled, it routes `/lens-ingest` directly to Lens. With a custom ingress, add that route yourself. For a dedicated hostname, use `lensWorker.ingress.enabled`, `host`, `className`, and `tls`, and set `publicUrl` to that hostname. The chart connects LiteLLM to Lens internally and gives both services the shared secret
Start with a source deployment that includes Lens, PostgreSQL, and agent tracing, and prepare its matching worker as described above. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries:
```yaml
general_settings:
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
```
The URL, database, and retention settings can also come from `CLICKHOUSE_URL`, `CLICKHOUSE_DATABASE`, and `AGENT_TRACING_RETENTION_DAYS` when omitted from YAML. A YAML value wins when both are set. The database defaults to `litellm`. `retention_days` defaults to 14 and applies to both traces and spend logs
Retention changes require a proxy restart. ClickHouse removes expired rows during background merges, not immediately at startup. Enable request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in
The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. Once the matching image is available on the worker host, no second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
The dashboard uses the gateway's `LENS_WORKER_IMAGE` override when set. Public `:sha-<commit>` development images must match both the gateway commit and release identity. Build from source for the worker host's native architecture
After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying
For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and an explicit `LENS_WORKER_IMAGE` in a private environment file:
Update your existing component image overrides to matching builds, then use the chart from that checkout:
```bash
docker compose --env-file /path/to/lens.env -f compose.yaml up -d
helm upgrade --install litellm ./helm/litellm \
--namespace litellm -f values.yaml --wait
```
To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
Use `./helm/litellm-helm` if that is your existing chart. `lensWorker.replicaCount` scales ingestion and investigations. Each replica needs access to the same ClickHouse and gateway. Credentials refresh every 30 seconds; a newly created key may briefly receive a retryable 429. Revocations propagate on refresh, and a replica stops accepting traces when its credential snapshot reaches 90 seconds
The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. Python reports storage failures to the reviewer and cleans up temporary files, so the reviewer can retry a smaller computation or report insufficient evidence. The worker remains available for other scans. Existing workers must be recreated with the new image and mount options
## Upgrade
The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles up to three investigations concurrently and can serve multiple lenses. For more throughput, start another worker with a separate credential
Upgrade LiteLLM and Lens from the same source commit and release identity. For a coordinated published release, use its matching worker version; `deploy/lens/stack.yaml` starts LiteLLM, Lens, PostgreSQL, and ClickHouse for new installations. Standalone images remain available. Publishing an image does not update running containers
If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
Keep the same databases, encryption keys, shared service secret, and public ingestion URL. Pause scheduled investigations and finish or cancel active runs, update both images through your usual deployment process, then check ingestion and run an investigation before resuming schedules. Do not run `docker compose down -v`
Setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
When upgrading from the Python worker, replace it with the Rust Lens service, move the existing ClickHouse connection to Lens, and configure the service URLs and secret on LiteLLM. Existing trace data remains in the same ClickHouse database; findings and settings remain in PostgreSQL. Stop the old worker. Generate dedicated tracing keys and change agent exporters to the ingestion URL. A virtual model key no longer authorizes uploads; the old gateway upload endpoints return 410 with setup guidance
If you retain an explicit `LENS_WORKER_TOKEN`, it remains an optional investigation credential. Normal setup uses the shared service connection and registers one managed worker identity. Configure the analysis model and billing key in the dashboard; provider keys stay on LiteLLM
## Development
`make lens-dev` starts LiteLLM, the Rust Lens service, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work
## Configure a lens
@ -116,7 +122,7 @@ Describe how the agent should behave and optionally add specific checks. Select
Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Each scan recalculates the window and reuses completed reviews when the selected trace content, expected behavior, enabled checks and analysis model are unchanged. Budget, name and schedule edits preserve reuse. Duplicate a lens when you want a separate investigation without changing an existing monitor
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 2 to 15 seconds, backing off while idle; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
## Read the results
@ -192,32 +198,14 @@ For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS
Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse variables before starting the proxy and seeder so both processes use the same settings. Invalid, zero and negative values fail instead of silently falling back. Changing these limits does not require rebuilding Rust
| Environment variable | Default | Controls |
| --- | --- | --- |
| `LENS_DEV_SEED_COPIES` | 1 default, 2000 large | Total fixture copies |
| `LENS_DEV_SEED_TIMEOUT_SECONDS` | 120 | Seeder HTTP timeout |
| `OTLP_MAX_BODY_BYTES` | 16777216 | HTTP body and decompressed payload bytes |
| `OTLP_MAX_CONCURRENT_INGESTS` | 2 | Concurrent proxy ingestion requests |
| `OTLP_MAX_ATTRIBUTE_VALUE_BYTES` | 65536 | Stored attribute/content bytes |
| `OTLP_MAX_DECODE_DEPTH` | 32 | Nested decode depth |
| `OTLP_MAX_DECODE_NODES` | 65536 | JSON values or protobuf fields per export |
| `OTLP_MAX_SPANS` | 4096 | Spans per export |
| `OTLP_MAX_ATTRIBUTES` | 256 | Attributes per resource, scope, span, event or link |
| `OTLP_MAX_EVENTS` | 256 | Events per span |
| `OTLP_MAX_LINKS` | 256 | Links per span |
| `OTLP_MAX_DECODED_SPAN_BYTES` | 16777216 | Decoded span allocation budget |
| `CLICKHOUSE_TRACE_MAX_INSERT_BYTES` | 67108864 | Encoded trace or spend insert bytes |
| `CLICKHOUSE_INSERT_TIMEOUT_SECONDS` | 30 | ClickHouse insert HTTP timeout |
The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits.
The Rust receiver bounds each upload and its decompressed body to 16 MiB and permits two ingestion requests at once per replica. Exporters should split large batches and retry backpressure. `LENS_DEV_SEED_COPIES` and `LENS_DEV_SEED_TIMEOUT_SECONDS` control the seeder; the receiver's limits are compiled into the service
## Quality evaluation
Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
```bash
cargo build --manifest-path litellm-rust/Cargo.toml -p litellm-lens --example worker_once --locked
python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
--model your-model-alias --split all --background 1000 --concurrency 16 \
--output /tmp/lens-quality.json
@ -250,7 +238,7 @@ The hourly development pipeline pins all component images to the same selected c
## Worker dependencies
The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures
The service builds from the workspace Cargo.lock with a pinned Rust toolchain and a digest-pinned Wolfi runtime. It has no Python package dependencies. CPython and libseccomp support the confined calculation tool. CI builds, runs, and scans native amd64 and arm64 images
## Python analysis boundary
@ -260,14 +248,14 @@ The native worker image builds a syscall policy with libseccomp and includes the
Python execution requires a native Linux worker with Landlock ABI 3 or later and seccomp filtering. Build the image for the host architecture. Missing policy files, an incompatible kernel, or an unsupported host such as a macOS source worker returns a clear tool error. There is no unrestricted execution fallback. Keep the container's non-root user, dropped capabilities, no-new-privileges setting, read-only root and writable temporary mount
The worker permits two Python children at once across all investigations. Set `LENS_PYTHON_CONCURRENCY` to a positive integer to change this worker-wide pool. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
The worker permits two Python children at once across all investigations. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
| Per-call resource | Default |
| --- | --- |
| Elapsed execution time | 60 seconds |
| CPU time | 30 seconds |
| Process address space | 512 MiB |
| Captured stdout or stderr | 8 MiB per stream |
| Captured stdout or stderr | 4 MiB per stream |
| Individual scratch file size | 16 MiB |
| Monitored scratch storage | 64 MiB |
| Monitored scratch entries | 2,048 |
@ -281,12 +269,11 @@ Results include `stdout`, `stderr`, `exit_code`, `error` and `output_complete`.
This is a process boundary sharing the worker's Linux kernel. The checked-in smoke test verifies useful Python operations, filesystem and process restrictions, raw syscall attempts, resource failures, mapping accounting, cleanup and cancellation in the actual image. Run it on the deployment's native architecture and kernel:
```bash
docker build --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-f deploy/lens/Dockerfile -t lens-worker:python-test .
docker run --rm --pull never --read-only --cap-drop ALL \
docker build --target smoke --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-f deploy/lens/Dockerfile -t lens-worker:smoke .
docker run --rm --read-only --cap-drop ALL \
--security-opt no-new-privileges --network none \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --entrypoint python -i \
lens-worker:python-test - < tests/proxy_behavior/lens/worker_python_smoke.py
--tmpfs /tmp:rw,noexec,nosuid,size=1g lens-worker:smoke
```
The same checks can run through pytest by setting `LENS_TEST_WORKER_IMAGE` to an already-built native image. The worker image CI runs the standalone smoke without adding pytest to the production image
The smoke target runs the Rust sandbox integration tests. The production image contains neither Cargo nor the test executable

12
deploy/lens/compose.yaml
View file

@ -3,8 +3,16 @@ services:
image: ${LENS_WORKER_IMAGE:-${LITELLM_VERSION:+ghcr.io/berriai/litellm-lens-worker:v}${LITELLM_VERSION:-}}
environment:
LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the same secret on LiteLLM and Lens}
CLICKHOUSE_URL: ${CLICKHOUSE_URL:?Set the ClickHouse URL reachable from Lens}
CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
ports:
- "127.0.0.1:${LENS_PORT:-4318}:4318"
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
read_only: true
tmpfs:

4
deploy/lens/config.yaml
View file

@ -2,6 +2,4 @@ general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
type: lens

2
deploy/lens/requirements.in
View file

@ -1,2 +0,0 @@
httpx==0.28.1
pydantic==2.13.4

172
deploy/lens/requirements.lock
View file

@ -1,172 +0,0 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock
annotated-types==0.8.0 \
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
# via pydantic
anyio==4.15.1 \
--hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
--hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
# via httpx
certifi==2026.7.22 \
--hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
--hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
# via
# httpcore
# httpx
h11==0.16.0 \
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
# via httpcore
httpcore==1.0.9 \
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
# via httpx
httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
# via -r deploy/lens/requirements.in
idna==3.20 \
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
# via
# anyio
# httpx
pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
# via -r deploy/lens/requirements.in
pydantic-core==2.46.4 \
--hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
--hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
--hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
--hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
--hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
--hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
--hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
--hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
--hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
--hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
--hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
--hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
--hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
--hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
--hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
--hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
--hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
--hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
--hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
--hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
--hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
--hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
--hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
--hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
--hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
--hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
--hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
--hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
--hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
--hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
--hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
--hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
--hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
--hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
--hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
--hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
--hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
--hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
--hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
--hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
--hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
--hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
--hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
--hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
--hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
--hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
--hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
--hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
--hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
--hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
--hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
--hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
--hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
--hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
--hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
--hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
--hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
--hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
--hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
--hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
--hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
--hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
--hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
--hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
--hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
--hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
--hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
--hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
--hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
--hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
--hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
--hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
--hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
--hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
--hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
--hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
--hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
--hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
--hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
--hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
--hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
--hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
--hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
--hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
--hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
--hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
--hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
--hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
--hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
--hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
--hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
--hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
--hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
--hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
--hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
--hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
--hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
--hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
--hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
--hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
--hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
--hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
--hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
--hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
--hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
--hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
--hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
--hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
--hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
--hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
--hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
--hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
--hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
--hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
--hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
--hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
--hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
--hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
--hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
--hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
# via pydantic
typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
# via
# anyio
# pydantic
# pydantic-core
# typing-inspection
typing-inspection==0.4.4 \
--hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \
--hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147
# via pydantic

28
deploy/lens/stack.yaml
View file

@ -10,9 +10,7 @@ services:
import os, sys
from urllib.parse import quote
postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="")
os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123"
os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
environment:
@ -20,29 +18,37 @@ services:
LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
STORE_MODEL_IN_DB: "True"
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
LITELLM_LENS_URL: http://lens-worker:4318
LITELLM_LENS_PUBLIC_URL: ${LITELLM_LENS_PUBLIC_URL:-http://localhost:4318}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the shared Lens service secret}
LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
volumes:
- ./config.yaml:/app/lens-config.yaml:ro
ports:
- "127.0.0.1:${LITELLM_PORT:-4000}:4000"
networks: [proxy, storage]
networks: [proxy, database]
depends_on:
db:
condition: service_healthy
clickhouse:
condition: service_healthy
restart: unless-stopped
lens-worker:
profiles: [lens]
image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
environment:
LITELLM_URL: http://litellm:4000
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN}
CLICKHOUSE_HOST: clickhouse
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
depends_on: [litellm]
networks: [proxy]
networks: [proxy, storage]
ports:
- "127.0.0.1:${LENS_PORT:-4318}:4318"
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
read_only: true
tmpfs:
@ -56,7 +62,7 @@ services:
POSTGRES_DB: litellm
POSTGRES_USER: litellm
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
networks: [storage]
networks: [database]
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
@ -84,6 +90,8 @@ services:
networks:
proxy:
database:
internal: true
storage:
internal: true

30
docker/docker-compose.tracing.yml
View file

@ -13,8 +13,9 @@ services:
LITELLM_SALT_KEY: sk-local-tracing-salt-key
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
STORE_MODEL_IN_DB: "True"
CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
CLICKHOUSE_DATABASE: litellm
LITELLM_LENS_URL: http://lens-worker:4318
LITELLM_LENS_PUBLIC_URL: http://localhost:4318
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
LENS_WORKER_IMAGE: ${LENS_WORKER_IMAGE:-}
volumes:
@ -24,8 +25,29 @@ services:
depends_on:
db:
condition: service_healthy
clickhouse:
condition: service_healthy
lens-worker:
build:
context: ..
dockerfile: deploy/lens/Dockerfile
args:
LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?set LITELLM_RELEASE_TAG to the source commit}
environment:
LITELLM_URL: http://litellm:4000
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
CLICKHOUSE_DATABASE: litellm
ports:
- "127.0.0.1:4318:4318"
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
tmpfs:
- /tmp:rw,noexec,nosuid,nodev,size=${LENS_WORKER_TMP_SIZE:-1g},mode=1777
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
db:
image: postgres:16

4
docker/tracing-config.yaml
View file

@ -8,6 +8,4 @@ general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
type: lens

50
helm/litellm-helm/templates/_helpers.tpl
View file

@ -321,3 +321,53 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- if .Values.lensWorker.image.digest -}}
{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}}
{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}}
{{- end -}}
{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}}
{{- else -}}
{{- $backendTag := .Values.image.tag | default .Chart.AppVersion -}}
{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
{{- $repository := .Values.lensWorker.image.repository -}}
{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}}
{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}}
{{- end -}}
{{- printf "%s:%s" $repository $tag -}}
{{- end -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
{{- end -}}
{{- end -}}
{{/*
LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the
consumer (collector container), so both agree on the transport and the
shutdown drain window.
*/}}
{{- define "litellm.gateway.collectorEnv" -}}
{{- with .Values.gateway.collector }}
- name: LITELLM_COLLECTOR_ENABLED
value: "true"
- name: LITELLM_COLLECTOR_ADDRESS
value: {{ .address | quote }}
- name: LITELLM_COLLECTOR_BUFFER_SIZE
value: {{ .bufferSize | quote }}
- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: {{ .onUnavailable | quote }}
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .drainTimeoutSeconds | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}

11
helm/litellm-helm/templates/deployment.yaml
View file

@ -56,6 +56,17 @@ spec:
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- if .Values.lensWorker.enabled }}
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- include "litellm.proxyEnv" . | nindent 12 }}
{{- if .Values.liteadmin.enabled }}
- name: LITELLM_ADMIN_AGENT_URL

14
helm/litellm-helm/templates/ingress.yaml
View file

@ -44,6 +44,20 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if $.Values.lensWorker.enabled }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}-lens-worker
port:
number: {{ $.Values.lensWorker.service.port }}
{{- else }}
serviceName: {{ $fullName }}-lens-worker
servicePort: {{ $.Values.lensWorker.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}

41
helm/litellm-helm/values.yaml
View file

@ -652,3 +652,44 @@ serviceMonitor:
namespaceSelector:
matchNames: []
# - test-namespace
lensWorker:
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/litellm-lens-worker
tag: ""
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
name: ""
key: token
serviceTokenSecret:
name: ""
key: service-token
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

20
helm/litellm/templates/_helpers.tpl
View file

@ -514,3 +514,23 @@ shutdown drain window.
value: {{ .drainTimeoutSeconds | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- if .Values.lensWorker.enabled }}
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}

1
helm/litellm/templates/backend/deployment.yaml
View file

@ -57,6 +57,7 @@ spec:
containerPort: 4001
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
- name: LENS_WORKER_IMAGE
value: {{ include "litellm.lensWorker.image" . | quote }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}

1
helm/litellm/templates/gateway/deployment.yaml
View file

@ -55,6 +55,7 @@ spec:
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH

10
helm/litellm/templates/ingress.yaml
View file

@ -156,6 +156,16 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if .Values.lensWorker.enabled }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ include "litellm.fullname" . }}-lens-worker
port:
number: {{ .Values.lensWorker.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---
Rendered after every built-in path so an entry can never take

33
helm/litellm/templates/lens/deployment.yaml
View file

@ -4,7 +4,7 @@ kind: Deployment
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- include "litellm.lensWorker.labels" . | nindent 4 }}
app.kubernetes.io/component: lens-worker
spec:
replicas: {{ .Values.lensWorker.replicaCount }}
@ -15,7 +15,7 @@ spec:
template:
metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
{{- include "litellm.lensWorker.labels" . | nindent 8 }}
app.kubernetes.io/component: lens-worker
spec:
automountServiceAccountToken: false
@ -42,11 +42,38 @@ spec:
env:
- name: LITELLM_URL
value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
- name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
- name: CLICKHOUSE_DATABASE
value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
- name: AGENT_TRACING_RETENTION_DAYS
value: {{ .Values.lensWorker.retentionDays | quote }}
{{- if .Values.lensWorker.tokenSecret.name }}
- name: LENS_WORKER_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }}
name: {{ .Values.lensWorker.tokenSecret.name | quote }}
key: {{ .Values.lensWorker.tokenSecret.key | quote }}
{{- end }}
ports:
- name: otlp
containerPort: 4318
livenessProbe:
httpGet:
path: /health/live
port: otlp
readinessProbe:
httpGet:
path: /health/ready
port: otlp
resources:
{{- toYaml .Values.lensWorker.resources | nindent 12 }}
volumeMounts:

40
helm/litellm/tests/lens_worker_tests.yaml
View file

@ -11,7 +11,9 @@ tests:
set:
backend.image.tag: sha-0123456789abcdef
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -31,7 +33,9 @@ tests:
set:
backend.image.tag: sha-0123456789abcdef
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.image.repository: registry.example/lens-worker
asserts:
- equal:
@ -41,7 +45,9 @@ tests:
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.image.tag: replaced-release
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
@ -71,20 +77,22 @@ tests:
asserts:
- hasDocuments:
count: 0
- it: requires a limited worker credential when enabled
- it: requires a shared service secret when enabled
template: lens/deployment.yaml
set:
lensWorker.enabled: true
asserts:
- failedTemplate:
errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token
errorMessage: lensWorker.serviceTokenSecret.name is required
- it: uses the chart release and a secret without granting Kubernetes access
template: lens/deployment.yaml
chart:
appVersion: v1.2.3
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -92,8 +100,8 @@ tests:
- equal:
path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
value:
name: lens-credential
key: token
name: lens-service
key: service-token
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
@ -120,7 +128,9 @@ tests:
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.url: https://gateway.example/proxy
lensWorker.image.repository: registry.example/lens-worker
lensWorker.image.tag: branch-main-1234567
@ -137,7 +147,9 @@ tests:
appVersion: 1.2.3-rc.4
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -147,7 +159,9 @@ tests:
set:
backend.image.tag: branch-main-1234567
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -167,7 +181,9 @@ tests:
set:
backend.image.tag: 1.2.3-dev.4
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image

18
helm/litellm/values.yaml
View file

@ -641,6 +641,24 @@ lensWorker:
tokenSecret:
name: ""
key: token
serviceTokenSecret:
name: ""
key: service-token
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:

8
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -417,6 +417,7 @@ model LiteLLM_MCPServerTable {
source_url String?
timeout Float?
max_concurrent_requests Int?
rpm Int?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
@ -1756,6 +1757,8 @@ model LiteLLM_AutoRouterDailySpend {
router_name String
router_type String
turns Int @default(0)
total_tokens BigInt @default(0)
token_recorded_turns Int @default(0)
spend Float @default(0)
saved_spend Float @default(0)
savings_estimated_turns Int @default(0)
@ -1969,6 +1972,11 @@ model LiteLLM_LensWorker {
data Json
}
model LiteLLM_LensIngestionKey {
id String @id
data Json
}
model LiteLLM_LensDataset {
id String
revision Int

133
litellm-rust/Cargo.lock generated
View file

@ -4171,6 +4171,45 @@ dependencies = [
"wiremock",
]
[[package]]
name = "litellm-lens"
version = "0.1.0"
dependencies = [
"axum",
"bytes",
"chrono",
"flate2",
"futures-util",
"http 1.4.2",
"jsonschema",
"libc",
"litellm-http",
"litellm-storage-clickhouse",
"litellm-traces",
"litellm-traces-cache",
"litellm-traces-clickhouse",
"litellm-tracing",
"prettyplease",
"prost",
"reqwest 0.12.28",
"rstest",
"serde",
"serde_json",
"sha2 0.10.9",
"subtle",
"syn 2.0.119",
"tempfile",
"thiserror 2.0.19",
"tokio",
"tower-http",
"tracing",
"typify",
"unicode-casefold",
"url",
"uuid",
"wiremock",
]
[[package]]
name = "litellm-llms"
version = "0.1.0"
@ -5409,6 +5448,16 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn 2.0.119",
]
[[package]]
name = "primeorder"
version = "0.13.6"
@ -5975,6 +6024,16 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "regress"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "158a764437582235e3501f683b93a0a6f8d825d04a789dbe5ed30b8799b8908a"
dependencies = [
"hashbrown 0.16.1",
"memchr",
]
[[package]]
name = "relative-path"
version = "1.9.3"
@ -6421,6 +6480,18 @@ dependencies = [
"parking_lot",
]
[[package]]
name = "schemars"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
dependencies = [
"dyn-clone",
"schemars_derive 0.8.22",
"serde",
"serde_json",
]
[[package]]
name = "schemars"
version = "0.9.0"
@ -6442,11 +6513,23 @@ dependencies = [
"chrono",
"dyn-clone",
"ref-cast",
"schemars_derive",
"schemars_derive 1.2.2",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals 0.29.1",
"syn 2.0.119",
]
[[package]]
name = "schemars_derive"
version = "1.2.2"
@ -6455,7 +6538,7 @@ checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"serde_derive_internals 0.30.0",
"syn 3.0.6",
]
@ -6561,6 +6644,17 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "serde_derive_internals"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "serde_derive_internals"
version = "0.30.0"
@ -7927,6 +8021,35 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "typify"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b715573a376585888b742ead9be5f4826105e622169180662e2c81bed4a149c3"
dependencies = [
"typify-impl",
]
[[package]]
name = "typify-impl"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7b026f540b148b81043c720889dbb942b08659aa8a43f624ac4f04dbfc1861"
dependencies = [
"heck",
"log",
"proc-macro2",
"quote",
"regress",
"schemars 0.8.22",
"semver",
"serde",
"serde_json",
"syn 2.0.119",
"thiserror 2.0.19",
"unicode-ident",
]
[[package]]
name = "ucd-trie"
version = "0.1.7"
@ -7951,6 +8074,12 @@ version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
[[package]]
name = "unicode-casefold"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f66b1c8f8caa2ab31dc6d3f35386f16efdab89668f93411e565ac368908e8f"
[[package]]
name = "unicode-general-category"
version = "1.1.0"

34
litellm-rust/crates/host-python/src/argument.rs
View file

@ -1,8 +1,5 @@
use pyo3::{prelude::*, types::PyDict};
/// The caller's own object for a public argument: the keyword if given, even an explicit
/// `None`, else the bound request's attribute. Every reader of a public Python call uses
/// this rule, so the callbacks and the provider see one object per argument.
pub fn lookup<'py>(
kwargs: &Bound<'py, PyDict>,
request: &Bound<'py, PyAny>,
@ -11,6 +8,9 @@ pub fn lookup<'py>(
if let Some(value) = kwargs.get_item(name)? {
return Ok(Some(value));
}
if let Ok(bound) = request.cast::<PyDict>() {
return bound.get_item(name);
}
request.getattr_opt(name)
}
@ -48,4 +48,32 @@ kwargs = {'api_key': key, 'api_base': None}
assert!(find("model").is_none());
});
}
#[rstest::rstest]
#[case::prepared_value("{'api_key': 'replacement'}", Some("replacement"))]
#[case::explicit_none("{'api_key': None}", None)]
#[case::bound_fallback("{}", Some("original"))]
fn prepared_mapping_overrides_bound_values(
#[case] source: &str,
#[case] expected: Option<&str>,
) {
crate::initialize_python();
Python::attach(|py| {
let bound = PyDict::new(py);
bound.set_item("api_key", "original").unwrap();
let source = std::ffi::CString::new(source).unwrap();
let prepared = py
.eval(&source, None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let value = lookup(&prepared, bound.as_any(), "api_key")
.unwrap()
.unwrap();
assert_eq!(
value.extract::<Option<String>>().unwrap().as_deref(),
expected
);
});
}
}

20
litellm-rust/crates/host-python/src/marshal.rs
View file

@ -28,9 +28,7 @@ pub fn to_py<T>(py: Python<'_>, value: &T) -> PyResult<Py<PyAny>>
where
T: Serialize + ?Sized,
{
pythonize::pythonize(py, value)
.map(Bound::unbind)
.map_err(PyErr::from)
Pythonized(value).into_pyobject(py).map(Bound::unbind)
}
pub fn json_object_field(py: Python<'_>, document: &str, name: &str) -> PyResult<Py<PyAny>> {
@ -114,13 +112,19 @@ mod tests {
});
}
#[test]
fn pythonized_maps_serializer_panics_to_a_base_exception() {
#[rstest::rstest]
#[case::wrapped(false)]
#[case::direct(true)]
fn output_conversion_maps_serializer_panics_to_a_base_exception(#[case] direct: bool) {
crate::initialize_python();
Python::attach(|py| {
let error = Pythonized(PanickingSerializer)
.into_pyobject(py)
.expect_err("serializer panic should become a Python exception");
let error = if direct {
to_py(py, &PanickingSerializer).unwrap_err()
} else {
Pythonized(PanickingSerializer)
.into_pyobject(py)
.unwrap_err()
};
assert!(error.is_instance_of::<PanicException>(py));
assert_eq!(error.to_string(), "PanicException: serializer panicked");
});

4
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -14,7 +14,9 @@ use litellm_secrets::source::SecretSource;
use std::sync::Arc;
pub use litellm_inference::RouteError as Error;
pub use types::{MessagesCall, MessagesCallResponse, MessagesShaping, messages_body};
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, messages_body,
};
#[derive(Clone)]
pub struct MessagesRoute {

32
litellm-rust/crates/inference-messages/src/prepare.rs
View file

@ -80,12 +80,13 @@ fn prepare_provider_request(
let sanitized = config.shape_request(
MessagesRequest { model, ..body },
shaping.reasoning_auto_summary,
shaping.settings.reasoning_auto_summary,
)?;
let trimmed = without_additional_drop_params(sanitized, &shaping.additional_drop_params)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let transformed = config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.drop_params),
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?;
let scoped =
@ -148,7 +149,7 @@ mod tests {
use serde_json::{Map, Value, json};
use super::*;
use crate::MessagesShaping;
use crate::{MessagesSettings, MessagesShaping};
#[fixture]
fn shaping() -> MessagesShaping {
@ -310,10 +311,13 @@ mod tests {
)
};
let shaping = MessagesShaping {
additional_drop_params: additional_drop_params
.iter()
.map(ToString::to_string)
.collect(),
settings: MessagesSettings {
additional_drop_params: additional_drop_params
.iter()
.map(ToString::to_string)
.collect(),
..shaping.settings
},
..shaping
};
assert_eq!(
@ -394,8 +398,11 @@ mod tests {
#[rstest]
fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
let shaping = MessagesShaping {
reasoning_auto_summary: true,
additional_drop_params: vec!["thinking.display".to_string()],
settings: MessagesSettings {
reasoning_auto_summary: true,
additional_drop_params: vec!["thinking.display".to_string()],
..shaping.settings
},
..shaping
};
assert_eq!(
@ -420,7 +427,10 @@ mod tests {
#[rstest]
fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
let shaping = MessagesShaping {
additional_drop_params: vec!["metadata.user_id".to_string()],
settings: MessagesSettings {
additional_drop_params: vec!["metadata.user_id".to_string()],
..shaping.settings
},
..shaping
};
assert!(matches!(

46
litellm-rust/crates/inference-messages/src/types.rs
View file

@ -38,6 +38,12 @@ pub type MessagesCallResponse =
pub struct MessagesShaping {
#[serde(default)]
pub capabilities: MessagesModelCapabilities,
#[serde(flatten)]
pub settings: MessagesSettings,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct MessagesSettings {
#[serde(default)]
pub drop_params: bool,
#[serde(default)]
@ -58,16 +64,25 @@ mod tests {
#[case::nothing_projected(json!({}), MessagesShaping::default())]
#[case::only_drop_params(
json!({"drop_params": true}),
MessagesShaping { drop_params: true, ..MessagesShaping::default() },
MessagesShaping {
settings: MessagesSettings { drop_params: true, ..MessagesSettings::default() },
..MessagesShaping::default()
},
)]
#[case::only_reasoning_auto_summary(
json!({"reasoning_auto_summary": true}),
MessagesShaping { reasoning_auto_summary: true, ..MessagesShaping::default() },
MessagesShaping {
settings: MessagesSettings { reasoning_auto_summary: true, ..MessagesSettings::default() },
..MessagesShaping::default()
},
)]
#[case::only_additional_drop_params(
json!({"additional_drop_params": ["tools[*].input_examples"]}),
MessagesShaping {
additional_drop_params: vec!["tools[*].input_examples".to_string()],
settings: MessagesSettings {
additional_drop_params: vec!["tools[*].input_examples".to_string()],
..MessagesSettings::default()
},
..MessagesShaping::default()
},
)]
@ -98,6 +113,11 @@ mod tests {
"additional_drop_params": ["metadata.user_id", "thinking"]
}),
MessagesShaping {
settings: MessagesSettings {
drop_params: true,
reasoning_auto_summary: true,
additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()],
},
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,
@ -115,9 +135,6 @@ mod tests {
max: false,
},
},
drop_params: true,
reasoning_auto_summary: true,
additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()],
},
)]
fn shaping_deserializes_with_defaults_for_absent_fields(
@ -126,5 +143,22 @@ mod tests {
) {
let shaping: MessagesShaping = serde_json::from_value(projected).unwrap();
assert_eq!(shaping, expected);
let serialized = serde_json::to_value(&shaping).unwrap();
assert_eq!(
serialized["drop_params"],
json!(expected.settings.drop_params)
);
assert_eq!(
serialized["reasoning_auto_summary"],
json!(expected.settings.reasoning_auto_summary)
);
assert_eq!(
serialized["additional_drop_params"],
json!(expected.settings.additional_drop_params)
);
assert_eq!(
serialized["capabilities"],
serde_json::to_value(expected.capabilities).unwrap()
);
}
}

6
litellm-rust/crates/inference-messages/tests/messages/host.rs
View file

@ -380,12 +380,14 @@ async fn the_request_context_carries_the_shaped_params_without_model_or_messages
let host = RecordingHost::passthrough(authenticated(
MessagesCall {
shaping: MessagesShaping {
settings: MessagesSettings {
drop_params: true,
..MessagesSettings::default()
},
capabilities: AnthropicModelCapabilities {
supports_sampling_params: false,
..AnthropicModelCapabilities::default()
},
drop_params: true,
..MessagesShaping::default()
},
..with_fields(call, json!({"temperature": 0.2}))
},

2
litellm-rust/crates/inference-messages/tests/messages/main.rs
View file

@ -5,7 +5,7 @@ use std::{
use litellm_http::{HttpSettings, Resolution};
use litellm_inference_messages::{
Error, MessagesCall, MessagesShaping,
Error, MessagesCall, MessagesSettings, MessagesShaping,
route::{Messages, MessagesMachine, MessagesOutput},
};
use litellm_inference_testing::RecordingSecrets;

27
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -239,7 +239,10 @@ async fn additional_drop_params_remove_fields_before_sending(call: MessagesCall)
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
additional_drop_params: vec!["temperature".into()],
settings: MessagesSettings {
additional_drop_params: vec!["temperature".into()],
..MessagesSettings::default()
},
..MessagesShaping::default()
},
..with_fields(call, json!({"temperature": 0.5, "top_k": 3}))
@ -390,8 +393,10 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
capabilities,
drop_params,
..MessagesShaping::default()
settings: MessagesSettings {
drop_params,
..MessagesSettings::default()
},
},
body: call.body.clone(),
custom_llm_provider: call.custom_llm_provider.clone(),
@ -436,13 +441,15 @@ async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
settings: MessagesSettings {
reasoning_auto_summary: true,
..MessagesSettings::default()
},
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,
..MessagesModelCapabilities::default()
},
reasoning_auto_summary: true,
..MessagesShaping::default()
},
..call
},
@ -634,7 +641,10 @@ async fn system_message_folding_is_selected_by_the_provider(
api_key: Some("sk-azure".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
additional_drop_params: drop_params.iter().map(ToString::to_string).collect(),
settings: MessagesSettings {
additional_drop_params: drop_params.iter().map(ToString::to_string).collect(),
..call.shaping.settings
},
..call.shaping
},
..call
@ -695,7 +705,10 @@ async fn provider_validation_runs_before_caller_parameter_removal(
api_key: Some("sk-test".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
additional_drop_params: vec!["metadata".into()],
settings: MessagesSettings {
additional_drop_params: vec!["metadata".into()],
..call.shaping.settings
},
..call.shaping
},
..call

18
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -29,10 +29,16 @@ pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr_above_100k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option<f64>,
@ -82,6 +88,10 @@ pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
@ -200,6 +210,10 @@ pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
@ -355,6 +369,10 @@ pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]

17
litellm-rust/crates/model-catalog/tests/registry_validation.rs
View file

@ -74,3 +74,20 @@ fn checked_in_catalog_and_backup_match() {
"invalid registry aliases"
);
}
#[rstest]
#[case::input("input_cost_per_token_above_100k_tokens")]
#[case::input_batches("input_cost_per_token_above_100k_tokens_batches")]
#[case::output("output_cost_per_token_above_100k_tokens")]
#[case::output_batches("output_cost_per_token_above_100k_tokens_batches")]
#[case::cache_creation("cache_creation_input_token_cost_above_100k_tokens")]
#[case::cache_creation_batches("cache_creation_input_token_cost_above_100k_tokens_batches")]
#[case::cache_creation_1hr("cache_creation_input_token_cost_above_1hr_above_100k_tokens")]
#[case::cache_read("cache_read_input_token_cost_above_100k_tokens")]
#[case::cache_read_batches("cache_read_input_token_cost_above_100k_tokens_batches")]
fn registry_validation_keeps_above_100k_tier_rates(#[case] field: &str) {
let mut entry = Map::new();
entry.insert("litellm_provider".into(), "anthropic".into());
entry.insert(field.into(), 5e-7.into());
validate_model_entry("test", &Value::Object(entry)).unwrap();
}

263
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -4,8 +4,9 @@ use std::{
};
use litellm_auth::InputSource;
use litellm_host_python::{from_py, from_py_argument};
use litellm_host_python::{from_py, from_py_argument, to_py};
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Map, Value};
/// The keyword arguments every value route shares, validated at the Python boundary.
@ -25,18 +26,6 @@ pub(crate) fn messages_argument(value: &Bound<'_, PyAny>) -> PyResult<Vec<Value>
}
}
pub(crate) fn optional_params_argument(
value: &Bound<'_, PyAny>,
) -> PyResult<Option<Map<String, Value>>> {
optional_object("optional_params", value)
}
pub(crate) fn extra_headers_argument(
value: &Bound<'_, PyAny>,
) -> PyResult<Option<Map<String, Value>>> {
optional_object("extra_headers", value)
}
fn required_object(name: &'static str, value: Value) -> PyResult<Map<String, Value>> {
match value {
Value::Object(values) => Ok(values),
@ -71,20 +60,77 @@ pub(crate) fn python_timeout_seconds(py: Python<'_>, timeout: Py<PyAny>) -> PyRe
.extract()
}
pub(crate) fn project_optional_fields(
kwargs: &Bound<'_, PyDict>,
names: &[&str],
pub(crate) fn required_field<'py>(
fields: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Bound<'py, PyAny>> {
fields
.get_item(name)?
.ok_or_else(|| PyValueError::new_err(format!("{name} is required")))
}
pub(crate) fn optional_field<T: DeserializeOwned>(
fields: &Bound<'_, PyDict>,
name: &str,
) -> PyResult<Option<T>> {
fields
.get_item(name)?
.map(|value| from_py_argument(&value))
.transpose()
.map(Option::flatten)
}
pub(crate) fn optional_object_field(
fields: &Bound<'_, PyDict>,
name: &'static str,
) -> PyResult<Option<Map<String, Value>>> {
fields
.get_item(name)?
.map(|value| optional_object(name, &value))
.transpose()
.map(Option::flatten)
}
pub(crate) fn value_route_options(fields: &Bound<'_, PyDict>) -> PyResult<RouteOptions> {
Ok(RouteOptions {
model: from_py_argument(&required_field(fields, "model")?)?,
api_key: optional_field(fields, "api_key")?,
api_base: optional_field(fields, "api_base")?,
custom_llm_provider: optional_field(fields, "custom_llm_provider")?,
extra_headers: optional_object_field(fields, "extra_headers")?,
timeout: optional_timeout(optional_field(fields, "timeout_seconds")?),
})
}
/// Builds a route's optional body fields from the caller's Python arguments, in `names` order.
/// `lookup` decides what counts as unset: a name it returns `None` for is left out of the map.
pub(crate) fn project_optional_fields<'a, 'py>(
names: impl IntoIterator<Item = &'a str>,
lookup: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
) -> PyResult<Map<String, Value>> {
names
.iter()
.filter_map(|name| match kwargs.get_item(name) {
Ok(Some(value)) => Some(from_py(&value).map(|value| ((*name).to_string(), value))),
.into_iter()
.filter_map(|name| match lookup(name) {
Ok(Some(value)) => Some(from_py(&value).map(|value| (name.to_string(), value))),
Ok(None) => None,
Err(error) => Some(Err(error)),
})
.collect()
}
/// Converts a Rust route response to Python and returns `module.response(...)` called on it,
/// so each route's Python factory builds the public LiteLLM response object.
pub(crate) fn public_response(
py: Python<'_>,
module: &str,
response: &(impl Serialize + ?Sized),
) -> PyResult<Py<PyAny>> {
py.import(module)?
.getattr("response")?
.call1((to_py(py, response)?,))
.map(Bound::unbind)
}
struct RequestFieldSources<'py> {
body: Option<Bound<'py, PyAny>>,
credentials: Option<Bound<'py, PyAny>>,
@ -152,6 +198,7 @@ pub(crate) fn marshal_headers(headers: Option<Value>) -> PyResult<HashMap<String
#[cfg(test)]
mod tests {
use pyo3::exceptions::PyTypeError;
use rstest::rstest;
use serde_json::json;
use super::*;
@ -162,6 +209,178 @@ mod tests {
locals
}
#[rstest]
#[case::keep_none(false)]
#[case::skip_none(true)]
fn selected_fields_preserve_lookup_order_and_caller_none_policy(#[case] skip_none: bool) {
Python::initialize();
Python::attach(|py| {
let locals = eval(
py,
c"
reads = []
fields = {'first': {'future': [None, True, 2]}, 'second': None, 'unused': object()}
def lookup(name):
reads.append(name)
if name == 'first':
fields['last'] = 'observed after first'
return fields.get(name)
",
);
let lookup = locals.get_item("lookup").unwrap().unwrap();
let result = project_optional_fields(["first", "second", "last"], |name| {
let value = lookup.call1((name,))?;
Ok((!skip_none || !value.is_none()).then_some(value))
})
.unwrap();
let expected = if skip_none {
json!({"first": {"future": [null, true, 2]}, "last": "observed after first"})
} else {
json!({"first": {"future": [null, true, 2]}, "second": null, "last": "observed after first"})
};
assert_eq!(Value::Object(result), expected);
assert_eq!(
locals
.get_item("reads")
.unwrap()
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["first", "second", "last"]
);
});
}
#[rstest]
fn selected_field_failure_stops_lookup_and_keeps_python_provenance() {
Python::initialize();
Python::attach(|py| {
let locals = eval(
py,
c"
reads = []
failure = LookupError('selected field failed')
cause = ValueError('cause')
def lookup(name):
reads.append(name)
raise failure from cause
",
);
let lookup = locals.get_item("lookup").unwrap().unwrap();
let error =
project_optional_fields(["first", "later"], |name| lookup.call1((name,)).map(Some))
.unwrap_err();
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
assert!(
error
.cause(py)
.unwrap()
.value(py)
.is(locals.get_item("cause").unwrap().unwrap())
);
assert!(error.traceback(py).is_some());
assert_eq!(
locals
.get_item("reads")
.unwrap()
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["first"]
);
});
}
#[rstest]
#[case::lookup_failure(false)]
#[case::factory_failure(true)]
fn public_response_resolves_factory_before_serializing_and_keeps_its_errors(
#[case] factory: bool,
) {
struct Observed<'a>(&'a std::cell::Cell<bool>);
impl Serialize for Observed<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
self.0.set(true);
json!({"future": [null, true]}).serialize(serializer)
}
}
Python::initialize();
Python::attach(|py| {
let module_name = if factory {
"bridge_response_conversion_test_factory"
} else {
"bridge_response_conversion_test_lookup"
};
let locals = eval(
py,
c"
import types
failure = LookupError('response failed')
cause = ValueError('cause')
received = []
def fail(name):
raise failure from cause
def response(value):
received.append(value)
return fail('response')
module = types.ModuleType('bridge_response_conversion_test')
module.__getattr__ = fail
",
);
py.import("sys")
.unwrap()
.getattr("modules")
.unwrap()
.set_item(module_name, locals.get_item("module").unwrap().unwrap())
.unwrap();
if factory {
locals
.get_item("module")
.unwrap()
.unwrap()
.setattr("response", locals.get_item("response").unwrap().unwrap())
.unwrap();
}
let serialized = std::cell::Cell::new(false);
let error = public_response(py, module_name, &Observed(&serialized)).unwrap_err();
assert_eq!(serialized.get(), factory);
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
assert!(
error
.cause(py)
.unwrap()
.value(py)
.is(locals.get_item("cause").unwrap().unwrap())
);
assert!(error.traceback(py).is_some());
let received: Value = from_py(&locals.get_item("received").unwrap().unwrap()).unwrap();
assert_eq!(
received,
if factory {
json!([{"future": [null, true]}])
} else {
json!([])
}
);
py.import("sys")
.unwrap()
.getattr("modules")
.unwrap()
.del_item(module_name)
.unwrap();
});
}
fn sources(
py: Python<'_>,
proxy: &Bound<'_, PyAny>,
@ -244,15 +463,15 @@ mod tests {
let params = py.eval(c"{'temperature': 0.2}", None, None).unwrap();
assert_eq!(
optional_params_argument(&params).unwrap(),
optional_object("optional_params", &params).unwrap(),
Some(required_object("optional_params", json!({"temperature": 0.2})).unwrap())
);
assert_eq!(
optional_params_argument(&py.None().into_bound(py)).unwrap(),
optional_object("optional_params", &py.None().into_bound(py)).unwrap(),
None
);
assert_eq!(
extra_headers_argument(&py.None().into_bound(py)).unwrap(),
optional_object("extra_headers", &py.None().into_bound(py)).unwrap(),
None
);
});

84
litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs
View file

@ -1,14 +1,13 @@
use crate::execution::{run_async, run_sync};
use litellm_host_python::from_py_argument;
use litellm_inference_transcription::{
AudioTranscriptionRoute, Error, types::AudioTranscriptionRequest,
};
use pyo3::{prelude::*, types::PyDict};
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::{
errors::route_error_to_pyerr,
marshal::{RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout},
marshal::{RouteOptions, optional_object_field, required_field, value_route_options},
};
async fn execute(
@ -41,81 +40,38 @@ async fn execute(
}
#[pyfunction]
#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))]
#[expect(
clippy::too_many_arguments,
reason = "one parameter per Python keyword"
)]
pub(crate) fn transcription(
py: Python<'_>,
model: String,
#[pyo3(from_py_with = from_py_argument)] audio: Value,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = extra_headers_argument)] extra_headers: Option<Map<String, Value>>,
#[pyo3(from_py_with = optional_params_argument)] optional_params: Option<Map<String, Value>>,
timeout_seconds: Option<f64>,
) -> PyResult<Py<PyAny>> {
let options = RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout: optional_timeout(timeout_seconds),
};
let http = crate::http::provider_client(py, &PyDict::new(py), false)?;
pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let http = crate::http::provider_client(py, &call.kwargs, false)?;
let secrets = crate::secrets::source(py)?;
run_sync(
py,
execute(
http,
secrets,
audio,
optional_params.unwrap_or_default(),
options,
),
execute(http, secrets, audio, optional_params, options),
route_error_to_pyerr,
)
}
#[pyfunction]
#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))]
#[expect(
clippy::too_many_arguments,
reason = "one parameter per Python keyword"
)]
pub(crate) fn atranscription<'py>(
py: Python<'py>,
model: String,
#[pyo3(from_py_with = from_py_argument)] audio: Value,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = extra_headers_argument)] extra_headers: Option<Map<String, Value>>,
#[pyo3(from_py_with = optional_params_argument)] optional_params: Option<Map<String, Value>>,
timeout_seconds: Option<f64>,
call: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let options = RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout: optional_timeout(timeout_seconds),
};
let http = crate::http::provider_client(py, &PyDict::new(py), true)?;
let call = super::NativeCall::extract(&call)?;
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let http = crate::http::provider_client(py, &call.kwargs, true)?;
let secrets = crate::secrets::source(py)?;
run_async(
py,
execute(
http,
secrets,
audio,
optional_params.unwrap_or_default(),
options,
),
execute(http, secrets, audio, optional_params, options),
route_error_to_pyerr,
)
}

100
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -11,8 +11,7 @@ use serde_json::{Map, Value};
use crate::{
errors::route_error_to_pyerr,
marshal::{
RouteOptions, extra_headers_argument, messages_argument, optional_params_argument,
optional_timeout,
RouteOptions, messages_argument, optional_object_field, required_field, value_route_options,
},
};
@ -50,81 +49,36 @@ async fn execute(
}
#[pyfunction]
#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
#[expect(
clippy::too_many_arguments,
reason = "one parameter per Python keyword"
)]
pub(crate) fn chat_completions(
py: Python<'_>,
model: String,
#[pyo3(from_py_with = messages_argument)] messages: Vec<Value>,
#[pyo3(from_py_with = optional_params_argument)] optional_params: Option<Map<String, Value>>,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = extra_headers_argument)] extra_headers: Option<Map<String, Value>>,
timeout_seconds: Option<f64>,
) -> PyResult<Py<PyAny>> {
let options = RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout: optional_timeout(timeout_seconds),
};
let http = crate::http::provider_client(py, &PyDict::new(py), false)?;
pub(crate) fn chat_completions(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, false)?;
let secrets = crate::secrets::source(py)?;
run_sync(
py,
execute(
http,
secrets,
messages,
optional_params.unwrap_or_default(),
options,
),
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
#[pyfunction]
#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
#[expect(
clippy::too_many_arguments,
reason = "one parameter per Python keyword"
)]
pub(crate) fn achat_completions<'py>(
py: Python<'py>,
model: String,
#[pyo3(from_py_with = messages_argument)] messages: Vec<Value>,
#[pyo3(from_py_with = optional_params_argument)] optional_params: Option<Map<String, Value>>,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = extra_headers_argument)] extra_headers: Option<Map<String, Value>>,
timeout_seconds: Option<f64>,
call: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let options = RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout: optional_timeout(timeout_seconds),
};
let http = crate::http::provider_client(py, &PyDict::new(py), true)?;
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, true)?;
let secrets = crate::secrets::source(py)?;
run_async(
py,
execute(
http,
secrets,
messages,
optional_params.unwrap_or_default(),
options,
),
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
@ -184,21 +138,13 @@ fn run_public(
}
#[pyfunction]
pub(crate) fn completion(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_public(py, request, args, kwargs, false)
pub(crate) fn completion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn acompletion(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_public(py, request, args, kwargs, true)
pub(crate) fn acompletion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, true)
}

61
litellm-rust/crates/python-bridge/src/routes/embeddings.rs
View file

@ -1,58 +1,49 @@
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use pyo3::prelude::*;
use crate::errors::RustBridgeDeclined;
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn embedding(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
pub(crate) fn embedding(call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
drop(super::NativeCall::extract(&call)?);
Err(RustBridgeDeclined::new_err(
"native embeddings route is not implemented",
))
}
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn aembedding(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native embeddings route is not implemented",
))
pub(crate) fn aembedding(call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
embedding(call)
}
#[cfg(test)]
mod tests {
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use pyo3::{prelude::*, types::PyDict};
use rstest::rstest;
use crate::errors::RustBridgeDeclined;
#[test]
fn both_entrypoints_decline_before_provider_execution() {
#[rstest]
#[case::sync(false)]
#[case::asynchronous(true)]
fn both_entrypoints_decline_before_provider_execution(#[case] asynchronous: bool) {
Python::initialize();
Python::attach(|py| {
let request = PyDict::new(py);
let args = PyTuple::empty(py);
let kwargs = PyDict::new(py);
for entrypoint in [super::embedding, super::aembedding] {
let error = entrypoint(request.clone().into_any(), args.clone(), kwargs.clone())
.expect_err("native embeddings must decline until a route machine exists");
assert!(error.is_instance_of::<RustBridgeDeclined>(py));
let locals = PyDict::new(py);
py.run(
c"from types import SimpleNamespace
call = SimpleNamespace(args=(), kwargs={}, bound={'model':'test-model','input':'hello'})",
Some(&locals),
Some(&locals),
)
.unwrap();
let call = locals.get_item("call").unwrap().unwrap();
let error = if asynchronous {
super::aembedding(call)
} else {
super::embedding(call)
}
.expect_err("native embeddings must decline until a route machine exists");
assert!(error.is_instance_of::<RustBridgeDeclined>(py));
});
}
}

26
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -1,5 +1,5 @@
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host_python::{from_py, lookup, to_py};
use litellm_host_python::{from_py, lookup};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
@ -8,7 +8,10 @@ use serde_json::{Map, Value};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{RouteOptions, optional_timeout, python_timeout_seconds},
marshal::{
RouteOptions, optional_timeout, project_optional_fields, public_response,
python_timeout_seconds,
},
};
pub(super) struct InferenceHost {
@ -86,6 +89,9 @@ impl InferenceHost {
if let Some(value) = lookup(arguments, request, name)? {
return Ok((!value.is_none()).then_some(value));
}
if request.is_instance_of::<PyDict>() {
return Ok(None);
}
let parameter = request
.getattr("parameters")?
.call_method1("get", (name,))?;
@ -102,21 +108,13 @@ impl InferenceHost {
arguments: &Bound<'_, PyDict>,
) -> PyResult<Map<String, Value>> {
let names: Vec<String> = py.import(self.module)?.getattr("PARAMETERS")?.extract()?;
names
.iter()
.filter_map(|name| match self.argument(py, arguments, name) {
Ok(Some(value)) => Some(from_py(&value).map(|value| (name.clone(), value))),
Ok(None) => None,
Err(error) => Some(Err(error)),
})
.collect()
project_optional_fields(names.iter().map(String::as_str), |name| {
self.argument(py, arguments, name)
})
}
pub fn response(&self, py: Python<'_>, response: &impl Serialize) -> PyResult<Py<PyAny>> {
py.import(self.module)?
.getattr("response")?
.call1((to_py(py, response)?,))
.map(Bound::unbind)
public_response(py, self.module, response)
}
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {

39
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -5,9 +5,10 @@ use bytes::Bytes;
use litellm_host_python::{InvokeError, PythonBinding, from_py, lookup, to_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference_messages::{
Error, MessagesCall, MessagesShaping, messages_body,
Error, MessagesCall, MessagesSettings, MessagesShaping, messages_body,
route::{Messages, MessagesStreamHead},
};
use litellm_llms::base_llm::messages::context::MessagesModelCapabilities;
use litellm_llms_types::headers::ProviderSpecificHeaders;
use pyo3::{
exceptions::{PyException, PyValueError},
@ -19,7 +20,7 @@ use serde_json::{Map, Value};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{optional_timeout, python_timeout_seconds},
marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds},
};
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host";
@ -115,14 +116,7 @@ impl MessagesPythonHost {
let model = string("model")?.ok_or_else(|| PyValueError::new_err("model is required"))?;
let messages =
argument("messages")?.ok_or_else(|| PyValueError::new_err("messages is required"))?;
let fields = BODY_FIELDS
.iter()
.filter_map(|name| match argument(name) {
Ok(Some(value)) => Some(from_py(&value).map(|value| ((*name).to_string(), value))),
Ok(None) => None,
Err(error) => Some(Err(error)),
})
.collect::<PyResult<Vec<(String, Value)>>>()?;
let fields = project_optional_fields(BODY_FIELDS, argument)?;
let body = [
("model".to_string(), Value::String(model.clone())),
("messages".to_string(), from_py(&messages)?),
@ -188,18 +182,24 @@ impl MessagesPythonHost {
custom_llm_provider: Option<&str>,
arguments: &Bound<'_, PyDict>,
) -> PyResult<MessagesShaping> {
let projected = py.import(ROUTE_HOST_MODULE)?.getattr("shaping")?.call1((
model,
custom_llm_provider,
arguments,
))?;
from_py(&projected)
let module = py.import(ROUTE_HOST_MODULE)?;
let capabilities: MessagesModelCapabilities = from_py(
&py.import("litellm.rust_bridge.model_capabilities")?
.getattr("anthropic_model_capabilities")?
.call1((model, custom_llm_provider))?,
)?;
let settings: MessagesSettings =
from_py(&module.getattr("settings")?.call1((arguments,))?)?;
Ok(MessagesShaping {
capabilities,
settings,
})
}
fn provider(&self, py: Python<'_>) -> String {
self.request
.bind(py)
.getattr("custom_llm_provider")
.get_item("custom_llm_provider")
.and_then(|value| value.extract::<Option<String>>())
.ok()
.flatten()
@ -249,10 +249,7 @@ impl PythonBinding for MessagesPythonHost {
py: Python<'_>,
response: Box<litellm_llms_types::formats::messages::MessagesResponse>,
) -> PyResult<Py<PyAny>> {
py.import(ROUTE_HOST_MODULE)?
.getattr("response")?
.call1((to_py(py, response.as_ref())?,))
.map(Bound::unbind)
public_response(py, ROUTE_HOST_MODULE, response.as_ref())
}
fn encode_stream_head(

20
litellm-rust/crates/python-bridge/src/routes/messages/mod.rs
View file

@ -64,21 +64,13 @@ fn run_messages(
}
#[pyfunction]
pub(crate) fn messages(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_messages(py, request, args, kwargs, false)
pub(crate) fn messages(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_messages(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn amessages(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_messages(py, request, args, kwargs, true)
pub(crate) fn amessages(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_messages(py, call.bound.into_any(), call.args, call.kwargs, true)
}

137
litellm-rust/crates/python-bridge/src/routes/mod.rs
View file

@ -14,9 +14,35 @@ use litellm_host::{call::HostedCompletion, machine::Machine, protocol::Protocol}
use litellm_host_python::{HookChain, PythonBinding, PythonCallHooks, PythonHostCalls};
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
types::{PyDict, PyMapping, PyTuple},
};
struct NativeCall<'py> {
args: Bound<'py, PyTuple>,
kwargs: Bound<'py, PyDict>,
bound: Bound<'py, PyDict>,
}
impl<'py> NativeCall<'py> {
fn extract(call: &Bound<'py, PyAny>) -> PyResult<Self> {
Ok(Self {
args: call.getattr("args")?.cast_into()?,
kwargs: mapping_dict(&call.getattr("kwargs")?)?,
bound: mapping_dict(&call.getattr("bound")?)?,
})
}
}
fn mapping_dict<'py>(value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyDict>> {
if let Ok(dict) = value.cast::<PyDict>() {
return Ok(dict.clone());
}
let mapping = value.cast::<PyMapping>()?;
let dict = PyDict::new(value.py());
dict.update(mapping)?;
Ok(dict)
}
fn call_hooks(
py: Python<'_>,
operation: LoggingOperation,
@ -74,40 +100,30 @@ mod tests {
types::{PyDict, PyList},
};
#[test]
fn sync_and_async_route_signatures_match_the_python_contract() {
Python::initialize();
Python::attach(|py| {
let module = crate::native_module(py);
let routes = [
(
"transcription",
"atranscription",
"(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
),
(
"chat_completions",
"achat_completions",
"(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
),
];
for (sync_name, async_name, expected) in routes {
let sync_signature: String = module
.getattr(sync_name)
.and_then(|function| function.getattr("__text_signature__"))
.and_then(|signature| signature.extract())
.expect("sync signature should be available");
let async_signature: String = module
.getattr(async_name)
.and_then(|function| function.getattr("__text_signature__"))
.and_then(|signature| signature.extract())
.expect("async signature should be available");
assert_eq!(sync_signature, expected);
assert_eq!(async_signature, expected);
}
});
fn value_call<'py>(
py: Python<'py>,
payload_name: &str,
payload: &Bound<'py, PyAny>,
kwargs: Option<&Bound<'py, PyDict>>,
) -> Bound<'py, PyAny> {
let fields = PyDict::new(py);
fields.set_item("model", "model").unwrap();
fields.set_item(payload_name, payload).unwrap();
if let Some(kwargs) = kwargs {
fields.update(kwargs.as_mapping()).unwrap();
}
let attributes = PyDict::new(py);
attributes
.set_item("args", pyo3::types::PyTuple::empty(py))
.unwrap();
attributes.set_item("kwargs", &fields).unwrap();
attributes.set_item("bound", &fields).unwrap();
py.import("types")
.unwrap()
.getattr("SimpleNamespace")
.unwrap()
.call((), Some(&attributes))
.unwrap()
}
#[test]
@ -138,7 +154,9 @@ value = Broken()
for name in ["chat_completions", "achat_completions"] {
let error = module
.getattr(name)
.and_then(|function| function.call1(("model", &broken)))
.and_then(|function| {
function.call1((value_call(py, "messages", &broken, None),))
})
.expect_err("route should reject a value it cannot convert");
assert!(
@ -158,11 +176,15 @@ value = Broken()
let invalid_messages = PyDict::new(py);
let sync_chat_error = module
.getattr("chat_completions")
.and_then(|function| function.call1(("model", &invalid_messages)))
.and_then(|function| {
function.call1((value_call(py, "messages", &invalid_messages, None),))
})
.expect_err("sync chat should reject a non-list messages value");
let async_chat_error = module
.getattr("achat_completions")
.and_then(|function| function.call1(("model", &invalid_messages)))
.and_then(|function| {
function.call1((value_call(py, "messages", &invalid_messages, None),))
})
.expect_err("async chat should reject a non-list messages value");
assert_eq!(
@ -180,11 +202,15 @@ value = Broken()
let sync_error = module
.getattr("transcription")
.and_then(|function| function.call(("model", &audio), Some(&kwargs)))
.and_then(|function| {
function.call1((value_call(py, "audio", &audio, Some(&kwargs)),))
})
.expect_err("sync route should reject non-dict extra_headers");
let async_error = module
.getattr("atranscription")
.and_then(|function| function.call(("model", &audio), Some(&kwargs)))
.and_then(|function| {
function.call1((value_call(py, "audio", &audio, Some(&kwargs)),))
})
.expect_err("async route should reject non-dict extra_headers");
assert_eq!(
@ -213,7 +239,12 @@ value = Broken()
let error = module
.getattr("chat_completions")
.and_then(|function| {
function.call(("model", &invalid_messages), Some(&chat_kwargs))
function.call1((value_call(
py,
"messages",
&invalid_messages,
Some(&chat_kwargs),
),))
})
.expect_err("messages should be validated first");
assert_eq!(error.to_string(), "ValueError: messages must be a list");
@ -221,7 +252,14 @@ value = Broken()
let valid_messages = PyList::empty(py);
let error = module
.getattr("chat_completions")
.and_then(|function| function.call(("model", &valid_messages), Some(&chat_kwargs)))
.and_then(|function| {
function.call1((value_call(
py,
"messages",
&valid_messages,
Some(&chat_kwargs),
),))
})
.expect_err("optional_params should be validated before headers");
assert_eq!(
error.to_string(),
@ -237,7 +275,12 @@ value = Broken()
let error = module
.getattr("transcription")
.and_then(|function| {
function.call(("model", &invalid_payload), Some(&headers_kwargs))
function.call1((value_call(
py,
"audio",
&invalid_payload,
Some(&headers_kwargs),
),))
})
.expect_err("payload should be validated before headers");
assert!(!error.to_string().contains("extra_headers"));
@ -265,11 +308,15 @@ value = Broken()
let omitted_error = module
.getattr("chat_completions")
.and_then(|function| function.call(("model", &messages), Some(&omitted)))
.and_then(|function| {
function.call1((value_call(py, "messages", &messages, Some(&omitted)),))
})
.expect_err("omitted optional_params should reach header validation");
let explicit_error = module
.getattr("chat_completions")
.and_then(|function| function.call(("model", &messages), Some(&explicit)))
.and_then(|function| {
function.call1((value_call(py, "messages", &messages, Some(&explicit)),))
})
.expect_err("None optional_params should reach header validation");
assert_eq!(
omitted_error.to_string(),

8
litellm-rust/crates/python-bridge/src/routes/ocr/host.rs
View file

@ -1,5 +1,5 @@
use litellm_auth::ResolvedCredential;
use litellm_host_python::{InvokeError, PythonBinding, missing_state, to_py};
use litellm_host_python::{InvokeError, PythonBinding, missing_state};
use litellm_host_python::{PythonHostCalls, PythonOwned};
use litellm_inference_ocr::route::{Ocr, OcrCall, OcrOp};
use litellm_llms::base_llm::ocr::error::Error;
@ -15,6 +15,7 @@ use super::{
errors::to_pyerr as ocr_error_to_pyerr,
project::{OcrHostHandles, project_request},
};
use crate::marshal::public_response;
enum OcrHostData {
Unprojected,
@ -104,10 +105,7 @@ impl PythonBinding for OcrPythonHost {
py: Python<'_>,
response: LiteLLMOcrResponse,
) -> PyResult<Py<PyAny>> {
py.import("litellm.rust_bridge.ocr.route_host")?
.getattr("response")?
.call1((to_py(py, &response)?,))
.map(Bound::unbind)
public_response(py, "litellm.rust_bridge.ocr.route_host", &response)
}
fn encode_stream_head(

20
litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs
View file

@ -81,23 +81,15 @@ fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult<OcrSettings> {
}
#[pyfunction]
pub(crate) fn ocr(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_ocr(py, request, args, kwargs, false)
pub(crate) fn ocr(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_ocr(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn aocr(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_ocr(py, request, args, kwargs, true)
pub(crate) fn aocr(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_ocr(py, call.bound.into_any(), call.args, call.kwargs, true)
}
#[pyfunction]

3
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -121,7 +121,8 @@ pub(super) fn project_request(
let specs = consumed_optional_params(&model, custom_llm_provider.as_deref())
.map_err(ocr_error_to_pyerr)?;
let names = specs.iter().map(|spec| spec.name).collect::<Vec<_>>();
let optional_params = project_optional_fields(kwargs, &names)?;
let optional_params =
project_optional_fields(names.iter().copied(), |name| kwargs.get_item(name))?;
let input_sources = request_input_sources(
kwargs,
names

20
litellm-rust/crates/python-bridge/src/routes/responses.rs
View file

@ -105,23 +105,15 @@ fn run_public(
}
#[pyfunction]
pub(crate) fn responses(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_public(py, request, args, kwargs, false)
pub(crate) fn responses(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn aresponses(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
run_public(py, request, args, kwargs, true)
pub(crate) fn aresponses(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, true)
}
#[pyclass]

9
litellm-rust/crates/router/tests/router.rs
View file

@ -1,7 +1,7 @@
use std::time::Duration;
use litellm_config::Config;
use litellm_inference_messages::MessagesShaping;
use litellm_inference_messages::{MessagesSettings, MessagesShaping};
use litellm_router::{Deployment, Router};
use rstest::rstest;
@ -74,8 +74,11 @@ fn programmatic_deployments_preserve_overrides_and_last_entry_wins() {
custom_llm_provider: Some("test-provider".into()),
timeout: Some(Duration::from_secs(7)),
shaping: MessagesShaping {
drop_params: true,
additional_drop_params: vec!["metadata.test".into()],
settings: MessagesSettings {
drop_params: true,
additional_drop_params: vec!["metadata.test".into()],
..MessagesSettings::default()
},
..Default::default()
},
};

3
litellm-rust/crates/traces-cache/src/cache.rs
View file

@ -325,6 +325,9 @@ mod tests {
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: String::new(),
api_key_hash: String::new(),
user_id: String::new(),

3
litellm-rust/crates/traces-cache/tests/read.rs
View file

@ -286,6 +286,9 @@ fn span(index: usize) -> TraceSpansRow {
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: "team".into(),
api_key_hash: "key".into(),
user_id: "user".into(),

3
litellm-rust/crates/traces-cache/tests/snapshots.rs
View file

@ -39,6 +39,9 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: String::new(),
api_key_hash: String::new(),
user_id: String::new(),

2
litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql
View file

@ -13,6 +13,8 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig
if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
AS tool_call_id,
o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url,
o.SpanAttributes['agent.source.title'] AS source_title,
o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
FROM otel_traces AS o
WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64})

2
litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql
View file

@ -12,6 +12,8 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig
if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
AS tool_call_id,
o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url,
o.SpanAttributes['agent.source.title'] AS source_title,
o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
FROM otel_traces AS o
WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64})

2
litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql
View file

@ -13,6 +13,8 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig
if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
AS tool_call_id,
o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url,
o.SpanAttributes['agent.source.title'] AS source_title,
o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
FROM otel_traces AS o
WHERE o.TraceId = {trace_id:String}

2
litellm-rust/crates/traces-clickhouse/query/trace_spans.sql
View file

@ -12,6 +12,8 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig
if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
AS tool_call_id,
o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url,
o.SpanAttributes['agent.source.title'] AS source_title,
o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
FROM otel_traces AS o
WHERE o.TraceId = {trace_id:String}

2
litellm-rust/crates/traces-clickhouse/src/lib.rs
View file

@ -16,6 +16,7 @@ mod insert;
pub mod query;
mod query_access;
mod reads;
mod receipt;
mod schema;
mod span_batches;
mod span_row;
@ -32,6 +33,7 @@ pub use litellm_traces::{QueryScope, ReadQuery};
pub use query::{QueryHelp, execute_read, query_help, query_sql};
pub use query_access::QueryReaders;
pub use reads::ClickHouseTraces;
pub use receipt::trace_received;
pub use schema::{
NORMALIZED_FIELD_DEFINITIONS, NormalizedFieldDefinition, apply_migrations, ensure_schema,
reconcile_retention, schema_statements,

64
litellm-rust/crates/traces-clickhouse/src/query/lens.rs
View file

@ -6,7 +6,8 @@ const SAMPLE_READ_LIMITS: ReadLimits = ReadLimits {
..litellm_storage_clickhouse::READ_LIMITS
};
pub const LENS_QUERIES: [litellm_traces::ReadQuery; 5] = [
pub const LENS_QUERIES: [litellm_traces::ReadQuery; 6] = [
litellm_traces::ReadQuery::TraceAgents,
litellm_traces::ReadQuery::Availability,
litellm_traces::ReadQuery::Agents,
litellm_traces::ReadQuery::Sample,
@ -107,6 +108,67 @@ impl Query for LensAgents {
const SQL: &'static str = include_str!("../../query/lens_agents.sql");
}
pub struct TraceAgents;
/// Same access shape as `list_traces`: every team, the caller's own traces, or their teams' traces.
#[macro_rules_attribute::apply(wire_type)]
#[derive(Debug)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
pub struct TraceAgentsParams {
#[serde(
deserialize_with = "super::number::boolean",
serialize_with = "litellm_traces::wire::serialize_flag"
)]
#[cfg_attr(
feature = "schema",
schemars(schema_with = "litellm_traces::schema::flag")
)]
pub all_teams: bool,
pub user_id: String,
pub team_ids: Vec<String>,
#[serde(deserialize_with = "super::number::deserialize")]
pub start_ms: i64,
#[serde(deserialize_with = "super::number::deserialize")]
pub end_ms: i64,
#[serde(deserialize_with = "super::number::deserialize")]
pub limit: u32,
}
#[macro_rules_attribute::apply(wire_type)]
#[derive(Debug)]
#[cfg_attr(feature = "schema", schemars(rename = "TraceAgentRow"))]
pub struct TraceAgentsRow {
pub agent_name: String,
#[serde(deserialize_with = "super::number::deserialize")]
#[cfg_attr(
feature = "schema",
schemars(schema_with = "crate::wire_schema::u64_number")
)]
pub runs: u64,
#[serde(deserialize_with = "super::number::deserialize")]
#[cfg_attr(
feature = "schema",
schemars(schema_with = "crate::wire_schema::u64_number")
)]
pub failed_runs: u64,
#[serde(deserialize_with = "super::number::deserialize")]
#[cfg_attr(
feature = "schema",
schemars(schema_with = "crate::wire_schema::u64_number")
)]
pub last_seen_ms: u64,
#[serde(default)]
pub frameworks: Vec<String>,
}
impl Query for TraceAgents {
type Params = TraceAgentsParams;
type Row = TraceAgentsRow;
const SQL: &'static str = include_str!("../../query/trace_agents.sql");
}
pub struct LensSample;
#[macro_rules_attribute::apply(wire_type)]

8
litellm-rust/crates/traces-clickhouse/src/query/named.rs
View file

@ -128,6 +128,12 @@ struct TraceSpansRowEncoding {
pub call_evidence: Option<litellm_traces::CallEvidenceKind>,
#[serde(default)]
pub tool_call_id: String,
#[serde(default)]
pub source_type: String,
#[serde(default)]
pub source_url: String,
#[serde(default)]
pub source_title: String,
pub team_id: String,
pub api_key_hash: String,
pub user_id: String,
@ -349,7 +355,7 @@ mod tests {
quoted,
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
quoted,
);
round_trip::<SpanDetailRow>(

1
litellm-rust/crates/traces-clickhouse/src/sql.rs
View file

@ -17,6 +17,7 @@ pub async fn execute_named_read(
) -> Result<String, Error> {
match query {
ReadQuery::ListTraces => named_json::<ListTraces>(client, connection, parameters).await,
ReadQuery::TraceAgents => named_json::<TraceAgents>(client, connection, parameters).await,
ReadQuery::TraceIdentity => {
named_json::<TraceIdentity>(client, connection, parameters).await
}

2
litellm-rust/crates/traces-clickhouse/src/wire_schema.rs
View file

@ -89,6 +89,8 @@ pub fn schemas() -> BTreeMap<&'static str, Schema> {
("PartRow", received::<lens::LensContentRow>()),
("CountRow", received::<lens::LensEvidenceRow>()),
("AgentRow", received::<lens::LensAgentsRow>()),
("TraceAgentsParams", received::<lens::TraceAgentsParams>()),
("TraceAgentRow", received::<lens::TraceAgentsRow>()),
("TraceQueryHelp", crate::query::help_schema()),
])
}

139
litellm-rust/crates/traces-clickhouse/tests/migrations.rs
View file

@ -754,6 +754,145 @@ async fn listed_agent_names_preserve_scope_and_cursor(
Ok(())
}
#[rstest]
#[tokio::test]
async fn trace_agents_count_runs_and_failures_within_scope_and_window(
#[future(awt)] database: TestResult<ClickHouseDatabase>,
) -> TestResult {
let database = database?;
let writer = Connection::writer(&database.url)?;
ensure_schema(&database.client, &writer, "trace_test", 7).await?;
let now = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
let old = now - 3 * 86_400_000_000_000_i64;
for (team, trace, span, parent, agent, status, framework, timestamp) in [
(
"alpha",
"run-1",
"root",
"",
"moyai",
"STATUS_CODE_OK",
"pi",
now,
),
(
"alpha",
"run-1",
"tool",
"root",
"moyai",
"STATUS_CODE_ERROR",
"pi",
now,
),
(
"alpha",
"run-2",
"root",
"",
"moyai",
"STATUS_CODE_OK",
"",
now - 1_000_000,
),
(
"alpha",
"run-3",
"root",
"",
"research",
"STATUS_CODE_OK",
"",
now - 2_000_000,
),
(
"alpha",
"old-run",
"root",
"",
"moyai",
"STATUS_CODE_ERROR",
"",
old,
),
(
"beta",
"other-team",
"root",
"",
"moyai",
"STATUS_CODE_ERROR",
"",
now,
),
(
"beta",
"other-agent",
"root",
"",
"hidden_agent",
"STATUS_CODE_OK",
"",
now,
),
] {
insert_rows(
&database,
"otel_traces",
vec![serde_json::from_value(serde_json::json!({
"Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent,
"ServiceName": "app", "SpanName": span, "AgentName": agent, "UserId": "owner",
"StatusCode": status, "Framework": framework, "ObservationType": "agent",
"ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": "key"}
}))?],
)
.await?;
}
let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
let parameters = BTreeMap::from([
("all_teams".into(), Parameter::Integer(0)),
("user_id".into(), Parameter::Text(String::new())),
("team_ids".into(), Parameter::Strings(vec!["alpha".into()])),
(
"start_ms".into(),
Parameter::Integer(now / 1_000_000 - 86_400_000),
),
("end_ms".into(), Parameter::Integer(now / 1_000_000 + 1000)),
("limit".into(), Parameter::Integer(10)),
]);
let agents: serde_json::Value = serde_json::from_str(
&execute_named_read(
&database.client,
&connection,
ReadQuery::TraceAgents,
&parameters,
)
.await?,
)?;
let rows = agents["data"].as_array().ok_or("missing agents")?;
let summary = rows
.iter()
.map(|row| {
(
row["agent_name"].as_str().unwrap_or_default(),
(
row["runs"].to_string().trim_matches('"').to_owned(),
row["failed_runs"].to_string().trim_matches('"').to_owned(),
row["frameworks"].clone(),
),
)
})
.collect::<Vec<_>>();
assert_eq!(
summary,
vec![
("moyai", ("2".into(), "1".into(), serde_json::json!(["pi"]))),
("research", ("1".into(), "0".into(), serde_json::json!([]))),
]
);
Ok(())
}
#[rstest]
#[tokio::test]
async fn rollup_merges_spans_across_days_without_losing_root_fields(

4
litellm-rust/crates/traces/src/lib.rs
View file

@ -44,6 +44,6 @@ pub use tenant::Tenant;
pub use truncate::{truncate_messages, truncate_value};
pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content};
pub use view::{
AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage,
TraceSummary,
AgentNode, RunSource, RunSourceType, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch,
Trace, TracePage, TraceSummary,
};

1
litellm-rust/crates/traces/src/query.rs
View file

@ -5,6 +5,7 @@ pub mod named;
#[strum(serialize_all = "snake_case")]
pub enum ReadQuery {
ListTraces,
TraceAgents,
TraceSpans,
TracePageSpans,
TraceIdentity,

6
litellm-rust/crates/traces/src/query/named.rs
View file

@ -110,6 +110,12 @@ pub struct TraceSpansRow {
pub call_evidence: Option<crate::CallEvidenceKind>,
#[serde(default)]
pub tool_call_id: String,
#[serde(default)]
pub source_type: String,
#[serde(default)]
pub source_url: String,
#[serde(default)]
pub source_title: String,
pub team_id: String,
pub api_key_hash: String,
pub user_id: String,

20
litellm-rust/crates/traces/src/resolve/view.rs
View file

@ -6,7 +6,9 @@ use time::OffsetDateTime;
use crate::{
normalize::ObservationType,
query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow},
view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary},
view::{
AgentNode, RunSource, RunSourceType, Span, SpanStatus, SpendMatch, Trace, TraceSummary,
},
};
use super::{
@ -138,6 +140,15 @@ pub fn iso_time(ms: i64) -> String {
)
}
fn source(row: &TraceSpansRow) -> Option<RunSource> {
row.source_url.starts_with("https://").then(|| RunSource {
kind: serde_json::from_value(serde_json::Value::from(row.source_type.as_str()))
.unwrap_or(RunSourceType::Custom),
url: row.source_url.clone(),
title: row.source_title.clone(),
})
}
fn sorted_unique<'a>(values: impl Iterator<Item = &'a str>) -> Vec<String> {
values
.filter(|value| !value.is_empty())
@ -229,6 +240,12 @@ pub fn resolve_trace(
models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())),
spend: priced.spend,
priced_calls: priced.priced_calls,
source: source(&rows[root]).or_else(|| {
rows.iter()
.filter_map(|row| Some((row.start_ns, source(row)?)))
.min_by_key(|(start_ns, _)| *start_ns)
.map(|(_, source)| source)
}),
};
Some(Trace {
summary,
@ -266,5 +283,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary {
models: row.models.clone(),
spend: None,
priced_calls: 0,
source: None,
}
}

27
litellm-rust/crates/traces/src/view.rs
View file

@ -66,6 +66,30 @@ pub struct AgentNode {
pub priced_calls: u64,
}
#[macro_rules_attribute::apply(wire_type)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum RunSourceType {
Slack,
Teams,
Discord,
Linear,
Github,
Jira,
#[serde(other)]
Custom,
}
/// The conversation that started the run, from the `agent.source.*` span attributes.
#[macro_rules_attribute::apply(response_type)]
#[derive(Clone, Debug, PartialEq)]
pub struct RunSource {
#[serde(rename = "type")]
pub kind: RunSourceType,
pub url: String,
pub title: String,
}
#[macro_rules_attribute::apply(response_type)]
#[derive(Clone, Debug, PartialEq)]
pub struct TraceSummary {
@ -95,6 +119,9 @@ pub struct TraceSummary {
pub models: Vec<String>,
pub spend: Option<f64>,
pub priced_calls: u64,
#[serde(skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))]
pub source: Option<RunSource>,
}
#[macro_rules_attribute::apply(response_type)]

6
litellm-rust/crates/traces/tests/captures.rs
View file

@ -207,6 +207,9 @@ fn trace_span(span: DecodedSpan) -> TraceSpansRow {
call_keys,
call_evidence: Some(normalized.calls.kind()),
tool_call_id: normalized.tool_call_id.unwrap_or_default(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: "fixture-team".into(),
api_key_hash: "fixture-key".into(),
user_id: "fixture-user".into(),
@ -291,6 +294,9 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow {
call_keys: vec![CallKey::Transport],
call_evidence: Some(CallEvidenceKind::Complete),
tool_call_id: String::new(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: call.team_id.clone(),
api_key_hash: call.api_key_hash.clone(),
user_id: call.user_id.clone(),

1
litellm-rust/crates/traces/tests/query.rs
View file

@ -3,6 +3,7 @@ use rstest::rstest;
#[rstest]
#[case::list_traces("list_traces", ReadQuery::ListTraces)]
#[case::trace_agents("trace_agents", ReadQuery::TraceAgents)]
#[case::trace_spans("trace_spans", ReadQuery::TraceSpans)]
#[case::span_detail("span_detail", ReadQuery::SpanDetail)]
#[case::span_error("span_error", ReadQuery::SpanError)]

2
litellm-rust/crates/traces/tests/query/named.rs
View file

@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() {
json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
);
round_trip::<SpanDetailRow>(
json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}),

56
litellm-rust/crates/traces/tests/resolve.rs
View file

@ -1,5 +1,5 @@
use litellm_traces::{
AgentNode, SpanStatus, SpendMatch, iso_time, listed_summary,
AgentNode, RunSourceType, SpanStatus, SpendMatch, iso_time, listed_summary,
query::named::{ListTracesRow, SpendByResponseIdsRow, TraceSpansRow},
resolve_trace,
};
@ -33,6 +33,9 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
source_type: String::new(),
source_url: String::new(),
source_title: String::new(),
team_id: "team".into(),
api_key_hash: "key".into(),
user_id: String::new(),
@ -173,6 +176,57 @@ fn summary_counts_model_calls_tools_and_agents() {
assert_eq!(summary.spend, None);
}
fn sourced(mut span: TraceSpansRow, url: &str, title: &str) -> TraceSpansRow {
span.source_url = url.into();
span.source_title = title.into();
span
}
const THREAD: &str = "https://acme.slack.com/archives/C1/p1";
#[rstest]
#[case::root_wins(
vec![sourced(at(row("root", "", "agent", "agent", "agent"), 5, 10), THREAD, "root thread"),
sourced(at(row("tool", "root", "tool", "tool", "agent"), 0, 1), "https://other.example/", "child")],
Some((THREAD, "root thread")),
)]
#[case::earliest_child_when_root_has_none(
vec![at(row("root", "", "agent", "agent", "agent"), 0, 10),
sourced(at(row("late", "root", "tool", "tool", "agent"), 5, 1), "https://late.example/", "late"),
sourced(at(row("early", "root", "tool", "tool", "agent"), 2, 1), THREAD, "early")],
Some((THREAD, "early")),
)]
#[case::non_https_is_dropped(
vec![sourced(row("root", "", "agent", "agent", "agent"), "javascript:alert(1)", "x")],
None,
)]
#[case::absent(vec![row("root", "", "agent", "agent", "agent")], None)]
fn summary_source_links_where_the_run_started(
#[case] rows: Vec<TraceSpansRow>,
#[case] expected: Option<(&str, &str)>,
) {
let source = resolve_trace("t", "", &rows, &[]).unwrap().summary.source;
assert_eq!(
source
.as_ref()
.map(|source| (source.url.as_str(), source.title.as_str())),
expected
);
}
#[rstest]
#[case::slack("slack", RunSourceType::Slack)]
#[case::teams("teams", RunSourceType::Teams)]
#[case::custom("custom", RunSourceType::Custom)]
#[case::unknown_is_custom("my-bot", RunSourceType::Custom)]
#[case::missing_is_custom("", RunSourceType::Custom)]
fn summary_source_type_picks_the_app(#[case] source_type: &str, #[case] expected: RunSourceType) {
let mut root = sourced(row("root", "", "agent", "agent", "agent"), THREAD, "t");
root.source_type = source_type.into();
let source = resolve_trace("t", "", &[root], &[]).unwrap().summary.source;
assert_eq!(source.map(|source| source.kind), Some(expected));
}
#[rstest]
fn spans_are_offset_from_the_trace_start() {
let trace = resolve_trace("t1", "", &deep_agent(1), &[]).unwrap();

34
litellm/chat_completions/dispatch.py
View file

@ -1,6 +1,5 @@
import inspect
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
@ -8,13 +7,13 @@ from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.chat_completions.entrypoints import (
NATIVE_ACOMPLETION,
NATIVE_COMPLETION,
LiteLLMChatCompletionsRequest,
)
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.public_call import (
NativeCall,
bind,
optional_bool,
optional_mapping,
native_call,
native_call_hook,
optional_sequence,
optional_str,
signature,
@ -51,33 +50,22 @@ _ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION)
def _public_request(
legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> LiteLLMChatCompletionsRequest | None:
) -> NativeCall | None:
fields: Final = bind(legacy, args, kwargs)
if fields is None:
return None
model: Final = fields.get("model")
messages: Final = optional_sequence(fields.get("messages"))
extra: Final = optional_mapping(fields.get("kwargs")) or MappingProxyType({})
if not isinstance(model, str) or messages is None:
return None
return LiteLLMChatCompletionsRequest(
model=model,
messages=messages,
stream=optional_bool(fields.get("stream")),
api_key=optional_str(fields.get("api_key")),
api_base=optional_str(extra.get("api_base")) or optional_str(fields.get("base_url")),
custom_llm_provider=optional_str(extra.get("custom_llm_provider")),
extra_headers=optional_mapping(fields.get("extra_headers")),
kwargs=extra,
parameters=MappingProxyType({name: value for name, value in fields.items() if name != "kwargs"}),
)
return native_call(args, kwargs, fields)
def _context(request: LiteLLMChatCompletionsRequest) -> RouteContext:
def _context(request: NativeCall) -> RouteContext:
return RouteContext(
Route.CHAT_COMPLETIONS,
provider=request.custom_llm_provider,
model=request.model,
provider=optional_str(request.bound.get("custom_llm_provider")),
model=str(request.bound["model"]),
)
@ -105,7 +93,7 @@ def completion(
kwargs,
python=python,
binding=NATIVE_COMPLETION,
native=call_hook,
native=native_call_hook,
)
@ -116,7 +104,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs-
kwargs,
python=python,
binding=NATIVE_ACOMPLETION,
native=call_hook,
native=native_call_hook,
)

72
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -24,6 +24,7 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_items_from_thinking_blocks,
@ -80,6 +81,38 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
)
def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
if not isinstance(value, dict):
return value
content_block: Final = cast(dict[str, object], value) # cast-ok: isinstance confirms the content block is a mapping
return {key: item for key, item in content_block.items() if key != "prompt_cache_breakpoint"}
def _strip_prompt_cache_breakpoints_from_content(value: object) -> object:
if isinstance(value, list):
list_content: Final = cast(list[object], value) # cast-ok: isinstance confirms a list of content blocks
return [_strip_prompt_cache_breakpoint_from_content_block(item) for item in list_content]
if isinstance(value, tuple):
tuple_content: Final = cast(tuple[object, ...], value) # cast-ok: isinstance confirms a tuple of content blocks
return tuple(_strip_prompt_cache_breakpoint_from_content_block(item) for item in tuple_content)
return _strip_prompt_cache_breakpoint_from_content_block(value)
def _strip_prompt_cache_breakpoints_from_item(value: object) -> object:
if not isinstance(value, dict):
return value
input_item: Final = cast(dict[str, object], value) # cast-ok: isinstance confirms a Responses input item mapping
return {
key: _strip_prompt_cache_breakpoints_from_content(item) if key in ("content", "output") else item
for key, item in input_item.items()
if key != "prompt_cache_breakpoint"
}
def _strip_prompt_cache_breakpoints(input_items: list[object]) -> list[object]:
return [_strip_prompt_cache_breakpoints_from_item(item) for item in input_items]
def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
return MappingProxyType(
{
@ -364,6 +397,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return None, index
def convert_chat_completion_messages_to_responses_api(
self,
messages: list["AllMessageValues"],
*,
keep_prompt_cache_breakpoints: bool = False,
) -> tuple[list[object], str | None]:
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(messages)
return (
converted_input_items
if keep_prompt_cache_breakpoints
else _strip_prompt_cache_breakpoints(converted_input_items),
instructions,
)
def _convert_chat_completion_messages_to_responses_input(
self, messages: list["AllMessageValues"]
) -> tuple[list[object], str | None]:
input_items: Final[list[object]] = []
@ -594,24 +641,31 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
litellm_logging_obj: "LiteLLMLoggingObj",
client: object | None = None,
) -> dict:
(
input_items,
instructions,
) = self.convert_chat_completion_messages_to_responses_api(messages)
base_model: Final = litellm_params.get("base_model")
supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model) or (
isinstance(base_model, str) and bool(base_model) and supports_openai_prompt_cache_breakpoint(base_model)
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
)
# OpenAI's Responses API rejects an empty input. For a system-only
# request, carry the system message as a system-role input item instead
# of instructions, mirroring how non-string system content is already
# handled in convert_chat_completion_messages_to_responses_api.
if not input_items and instructions is not None:
input_items = [
is_system_only_request: Final = not converted_input_items and converted_instructions is not None
input_items: Final = (
[
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": instructions}],
"content": [{"type": "input_text", "text": converted_instructions}],
}
]
instructions = None
if is_system_only_request
else converted_input_items
)
instructions: Final = None if is_system_only_request else converted_instructions
optional_params = self._extract_extra_body_params(optional_params)

1
litellm/constants.py
View file

@ -60,6 +60,7 @@ TRACE_READ_RETRY_AFTER_SECONDS: Final = get_env_int("TRACE_READ_RETRY_AFTER_SECO
OTLP_MAX_CONCURRENT_INGESTS: Final = get_env_int("OTLP_MAX_CONCURRENT_INGESTS", 2)
AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240)
AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50)
AGENT_TRACING_AGENT_LIST_LIMIT: Final = get_env_int("AGENT_TRACING_AGENT_LIST_LIMIT", 500)
LENS_DATASET_MAX_CASES: Final = get_env_int("LENS_DATASET_MAX_CASES", 200)
LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", 20_000)
LENS_DATASET_TRACE_PAGE_SIZE: Final = 500

32
litellm/embeddings/dispatch.py
View file

@ -1,17 +1,15 @@
import inspect
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.embeddings.entrypoints import (
NATIVE_AEMBEDDING,
NATIVE_EMBEDDING,
LiteLLMEmbeddingRequest,
)
from litellm.rust_bridge.public_call import bind, optional_mapping, optional_str, signature
from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature
from litellm.types.utils import EmbeddingResponse
__all__ = ("aembedding", "embedding")
@ -30,28 +28,24 @@ _EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
def _public_request(
legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> LiteLLMEmbeddingRequest | None:
) -> NativeCall | None:
fields: Final = bind(legacy, args, kwargs)
if fields is None:
return None
model: Final = fields.get("model")
if not isinstance(model, str):
return None
extra: Final = optional_mapping(fields.get("kwargs")) or MappingProxyType({})
return LiteLLMEmbeddingRequest(
model=model,
input=fields.get("input"),
api_key=optional_str(fields.get("api_key")),
api_base=optional_str(fields.get("api_base")),
custom_llm_provider=optional_str(fields.get("custom_llm_provider")),
kwargs=extra,
return native_call(args, kwargs, fields)
def _context(request: NativeCall) -> RouteContext:
return RouteContext(
Route.EMBEDDINGS,
provider=optional_str(request.bound.get("custom_llm_provider")),
model=str(request.bound["model"]),
)
def _context(request: LiteLLMEmbeddingRequest) -> RouteContext:
return RouteContext(Route.EMBEDDINGS, provider=request.custom_llm_provider, model=request.model)
_DISPATCH: Final = PublicDispatch(
route=Route.EMBEDDINGS,
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
@ -75,7 +69,7 @@ def embedding(
kwargs,
python=_PYTHON_EMBEDDING,
binding=NATIVE_EMBEDDING,
native=call_hook,
native=native_call_hook,
)
@ -85,7 +79,7 @@ async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # k
kwargs,
python=_PYTHON_AEMBEDDING,
binding=NATIVE_AEMBEDDING,
native=call_hook,
native=native_call_hook,
)

77
litellm/experimental_mcp_client/client.py
View file

@ -7,6 +7,7 @@ import base64
import hashlib
import json
import os
import time
from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Sequence
from contextlib import AbstractAsyncContextManager
from functools import partial
@ -35,6 +36,7 @@ _TransportContext: TypeAlias = AbstractAsyncContextManager[_TransportStreams]
from mcp.types import (
METHOD_NOT_FOUND,
REQUEST_TIMEOUT,
CacheableResult,
ClientCapabilities,
DiscoverResult,
ElicitationCapability,
@ -55,7 +57,6 @@ from mcp.types import (
ListToolsRequest,
ListToolsResult,
PaginatedRequestParams,
PaginatedResult,
Prompt,
ResourceTemplate,
SamplingCapability,
@ -77,7 +78,11 @@ from litellm.constants import (
from litellm.experimental_mcp_client.tools import list_tools_with_pagination
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response
from litellm.proxy._experimental.mcp_server.result_conversion import error_text_result
from litellm.proxy._experimental.mcp_server.result_conversion import (
age_freshness,
aggregate_freshness,
error_text_result,
)
from litellm.types.llms.custom_http import VerifyTypes
from litellm.types.mcp import (
MCP_LEGACY_VERSIONS,
@ -178,7 +183,7 @@ def as_mcp_read_timeout(exc: BaseException) -> TimeoutError | None:
TSessionResult = TypeVar("TSessionResult")
_ListPage = TypeVar("_ListPage", bound=PaginatedResult)
_ListPage = TypeVar("_ListPage", ListPromptsResult, ListResourcesResult, ListResourceTemplatesResult)
_ListItem = TypeVar("_ListItem")
@ -1031,12 +1036,21 @@ class MCPClient:
# Return a default error result instead of raising
return self.error_tool_result(e)
async def _run_optional_discovery(self, operation: Callable[[ClientSession], Awaitable[_ListPage]]) -> _ListPage:
async def timed_operation(session: ClientSession) -> tuple[_ListPage, float]:
result: Final = await operation(session)
return result, time.monotonic()
result, received = await self.run_with_session(timed_operation)
return age_freshness(result, time.monotonic() - received)
async def _list_optional_pages(
self,
fetch_page: Callable[[PaginatedRequestParams | None], Awaitable[_ListPage]],
items_of: Callable[[_ListPage], Sequence[_ListItem]],
) -> list[_ListItem]: # mutable-ok: existing list discovery API
) -> tuple[list[_ListItem], CacheableResult]:
items: Final[list[_ListItem]] = [] # mutable-ok: bounded iterative page accumulation
pages: Final[list[tuple[CacheableResult, float]]] = [] # mutable-ok: bounded pagination evidence
cursors: Final[set[str]] = set() # mutable-ok: constant-time detection of cursor cycles
cursor: str | None = None # rebind-ok: iterative traversal avoids recursion at the existing page cap
with anyio.fail_after(max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)):
@ -1047,9 +1061,13 @@ class MCPClient:
if page_index > 0 and error.error.code == METHOD_NOT_FOUND:
raise RuntimeError("MCP list operation became unavailable during pagination") from error
raise
pages.append((page, time.monotonic()))
items.extend(items_of(page))
if not page.next_cursor:
return items
now: Final = time.monotonic()
return items, aggregate_freshness(
tuple(age_freshness(value, now - received) for value, received in pages)
)
if page.next_cursor in cursors:
raise RuntimeError("MCP list pagination repeated a cursor")
cursors.add(page.next_cursor)
@ -1057,6 +1075,9 @@ class MCPClient:
raise RuntimeError(f"MCP list pagination exceeded {MCP_TOOL_LISTING_MAX_PAGES} pages")
async def list_prompts(self, *, raise_on_error: bool = False) -> list[Prompt]:
return (await self.list_prompts_result(raise_on_error=raise_on_error)).prompts
async def list_prompts_result(self, *, raise_on_error: bool = False) -> ListPromptsResult:
"""List available prompts from the server."""
verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio")
@ -1065,11 +1086,10 @@ class MCPClient:
if capabilities is not None and capabilities.prompts is None:
return ListPromptsResult(prompts=[])
try:
return ListPromptsResult(
prompts=await self._list_optional_pages(
lambda params: session.list_prompts(params=params), lambda page: page.prompts
)
items, freshness = await self._list_optional_pages(
lambda params: session.list_prompts(params=params), lambda page: page.prompts
)
return ListPromptsResult(prompts=items, ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope)
except MCPError as error:
if error.error.code != METHOD_NOT_FOUND:
raise
@ -1079,13 +1099,13 @@ class MCPClient:
return ListPromptsResult(prompts=[])
try:
result: Final = await self.run_with_session(_list_prompts_operation)
result: Final = await self._run_optional_discovery(_list_prompts_operation)
prompt_count: Final = len(result.prompts)
prompt_names: Final = [prompt.name for prompt in result.prompts]
verbose_logger.info(
"MCP client listed %s tools from %s: %s", prompt_count, self.server_url or "stdio", prompt_names
)
return result.prompts
return result
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_prompts was cancelled")
raise
@ -1107,7 +1127,7 @@ class MCPClient:
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
return ListPromptsResult(prompts=[])
async def get_prompt(self, get_prompt_request_params: GetPromptRequestParams) -> GetPromptResult:
"""Fetch a prompt definition from the MCP server."""
@ -1151,6 +1171,9 @@ class MCPClient:
raise
async def list_resources(self, *, raise_on_error: bool = False) -> list[Resource]:
return (await self.list_resources_result(raise_on_error=raise_on_error)).resources
async def list_resources_result(self, *, raise_on_error: bool = False) -> ListResourcesResult:
"""List available resources from the server."""
verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio")
@ -1159,11 +1182,10 @@ class MCPClient:
if capabilities is not None and capabilities.resources is None:
return ListResourcesResult(resources=[])
try:
return ListResourcesResult(
resources=await self._list_optional_pages(
lambda params: session.list_resources(params=params), lambda page: page.resources
)
items, freshness = await self._list_optional_pages(
lambda params: session.list_resources(params=params), lambda page: page.resources
)
return ListResourcesResult(resources=items, ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope)
except MCPError as error:
if error.error.code != METHOD_NOT_FOUND:
raise
@ -1173,13 +1195,13 @@ class MCPClient:
return ListResourcesResult(resources=[])
try:
result: Final = await self.run_with_session(_list_resources_operation)
result: Final = await self._run_optional_discovery(_list_resources_operation)
resource_count: Final = len(result.resources)
resource_names: Final = [resource.name for resource in result.resources]
verbose_logger.info(
"MCP client listed %s resources from %s: %s", resource_count, self.server_url or "stdio", resource_names
)
return result.resources
return result
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resources was cancelled")
raise
@ -1201,9 +1223,12 @@ class MCPClient:
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
return ListResourcesResult(resources=[])
async def list_resource_templates(self, *, raise_on_error: bool = False) -> list[ResourceTemplate]:
return (await self.list_resource_templates_result(raise_on_error=raise_on_error)).resource_templates
async def list_resource_templates_result(self, *, raise_on_error: bool = False) -> ListResourceTemplatesResult:
"""List available resource templates from the server."""
verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio")
@ -1212,11 +1237,11 @@ class MCPClient:
if capabilities is not None and capabilities.resources is None:
return ListResourceTemplatesResult(resource_templates=[])
try:
items, freshness = await self._list_optional_pages(
lambda params: session.list_resource_templates(params=params), lambda page: page.resource_templates
)
return ListResourceTemplatesResult(
resource_templates=await self._list_optional_pages(
lambda params: session.list_resource_templates(params=params),
lambda page: page.resource_templates,
)
resource_templates=items, ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope
)
except MCPError as error:
if error.error.code != METHOD_NOT_FOUND:
@ -1227,7 +1252,7 @@ class MCPClient:
return ListResourceTemplatesResult(resource_templates=[])
try:
result: Final = await self.run_with_session(_list_resource_templates_operation)
result: Final = await self._run_optional_discovery(_list_resource_templates_operation)
resource_template_count: Final = len(result.resource_templates)
resource_template_names: Final = [resource_template.name for resource_template in result.resource_templates]
verbose_logger.info(
@ -1236,7 +1261,7 @@ class MCPClient:
self.server_url or "stdio",
resource_template_names,
)
return result.resource_templates
return result
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resource_templates was cancelled")
raise
@ -1258,7 +1283,7 @@ class MCPClient:
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
return ListResourceTemplatesResult(resource_templates=[])
async def read_resource(self, url: AnyUrl) -> ReadResourceResult:
"""Fetch resource contents from the MCP server."""

8
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -21,7 +21,7 @@ from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.tracing.types import SpendLogRecord
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
from litellm.types.utils import StandardLoggingPayload
# litellm_logging.py rewrites cache-hit ids as f"{id}_cache_hit{time.time()}"
@ -115,7 +115,7 @@ def _request_tags(value: object) -> list[str]:
return [str(tag) for tag in value]
def _session_id(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> str:
def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]) -> str:
"""Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id."""
request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({})
return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "")
@ -126,7 +126,9 @@ def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
return str(payload.get("call_type") or "").startswith(TRACE_INGEST_ROUTE)
def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> SpendLogRecord:
def spend_log_row_from_payload(
payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]
) -> SpendLogRecord:
metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({})
hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({})
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})

3
litellm/integrations/otel/model/spans.py
View file

@ -352,6 +352,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_LensDataset",
"LiteLLM_LensRun",
"LiteLLM_LensReview",
"LiteLLM_LensIngestionKey",
"LiteLLM_LensWorker",
"LiteLLM_LensSignalConfig",
"LiteLLM_LensTraceSignal",
@ -474,7 +475,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
_RAW_PRISMA_CALL_TYPES: Final[frozenset[str]] = frozenset(("query_raw", "execute_raw"))
_DB_OPERATION_METADATA_KEY: Final = "db_operation"
_POSTGRES_VERBS: Final[frozenset[str]] = frozenset(
("select", "insert", "update", "delete", "upsert", "ddl", "set", "ping")
("select", "insert", "update", "delete", "upsert", "ddl", "set", "ping", "lock")
)
_TARGETLESS_VERBS: Final[frozenset[str]] = frozenset(("ping",))
_SETTING_NAME: Final = re.compile(r"[a-z_][a-z0-9_.]*")

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -291,7 +291,7 @@ else:
_GENERIC_API_LOGGER_CLS: Final = GenericAPILogger
_in_memory_loggers: Final[list[CustomLogger]] = []
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token",))
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object"))
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
)
@ -5993,7 +5993,7 @@ class StandardLoggingPayloadSetup:
Like get_usage_from_response_obj but returns a plain dict, skipping
the Pydantic Usage construction on the hot path.
"""
_empty: Final[dict] = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
_empty: Final[dict[str, object]] = {}
if combined_usage_object is not None:
return combined_usage_object.model_dump()
if not response_obj:

20
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2303,9 +2303,29 @@ class CustomStreamWrapper:
429 (rate-limit) is explicitly exempted from the 4xx filter because
it is transient and the Router should switch to another model group.
An error an inner stream already wrapped (the chat-to-Responses bridge
consumes a Responses stream) is rebuilt around the provider exception
with this wrapper's own bookkeeping, so the Router's one-level unwrap
surfaces the provider exception and is_pre_first_chunk says whether
this wrapper's consumer received anything (the inner stream counts a
lifecycle event the bridge never forwards as its first chunk).
"""
from litellm.exceptions import MidStreamFallbackError
if isinstance(e, MidStreamFallbackError):
self._restore_consumer_correlation_context()
if e.original_exception is None:
raise e
raise MidStreamFallbackError(
message=str(e.original_exception),
model=self.model,
llm_provider=self.custom_llm_provider or "anthropic",
original_exception=e.original_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
)
# Map to OpenAI exception format. Some providers' mappers (e.g.
# _map_anthropic_exception, _map_aleph_alpha_exception) synchronously
# log a debug diagnostic (the raw status code) as part of mapping -

3
litellm/llms/anthropic/count_tokens/handler.py
View file

@ -12,6 +12,7 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
@ -62,7 +63,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Processing Anthropic CountTokens request for model: %s", model)
# Transform request to Anthropic format
request_body: Final = self.transform_request_to_count_tokens(
request_body: Final = await asyncify(self.transform_request_to_count_tokens)(
model=model,
messages=messages,
tools=tools,

32
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -13,11 +13,41 @@ from pydantic import JsonValue, TypeAdapter
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import ChatCompletionImageObject
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
_IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject)
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
def _count_image(block: JsonValue) -> JsonValue:
if not isinstance(block, dict) or block.get("type") != "image_url":
return block
from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj
image_block: Final = _IMAGE_BLOCK.validate_python(block)
image_url: Final = image_block["image_url"]
source: Final = convert_to_anthropic_image_obj(
openai_image_url=image_url if isinstance(image_url, str) else image_url["url"],
format=image_url.get("format") if isinstance(image_url, dict) else None,
)
image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source})
return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image}
def _count_block(block: JsonValue) -> JsonValue:
if not isinstance(block, dict) or block.get("type") != "tool_result":
return _count_image(block)
content: Final = block.get("content")
if not isinstance(content, list):
return block
return {**block, "content": [_count_image(part) for part in content]}
def _count_content(content: JsonValue) -> JsonValue:
return [_count_block(block) for block in content] if isinstance(content, list) else content
class AnthropicCountTokensConfig:
"""
Configuration and transformation logic for Anthropic CountTokens API.
@ -62,7 +92,7 @@ class AnthropicCountTokensConfig:
MappingProxyType(
{
"model": model,
"messages": messages,
"messages": [{**message, "content": _count_content(message["content"])} for message in messages],
**MappingProxyType(
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
),

3
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -10,6 +10,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
@ -59,7 +60,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
verbose_logger.debug("Processing Azure AI Anthropic CountTokens request for model: %s", model)
# Transform request to Anthropic format
request_body: Final = self.transform_request_to_count_tokens(
request_body: Final = await asyncify(self.transform_request_to_count_tokens)(
model=model,
messages=messages,
tools=tools,

49
litellm/llms/bedrock/audio_transcription/__init__.py
View file

@ -6,6 +6,7 @@ import httpx
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.public_call import NativeCall
from litellm.rust_bridge.timeouts import timeout_to_seconds
from litellm.rust_bridge.transcription.native import (
NATIVE_ATRANSCRIPTION,
@ -52,18 +53,18 @@ class BedrockAudioTranscriptionRustDispatch:
timeout: float | httpx.Timeout | None,
) -> TranscriptionResponse:
def native(rust: RustTranscription) -> TranscriptionResponse:
return TranscriptionResponse(
**rust(
model=model,
audio=self._audio_payload(audio_file),
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
optional_params=optional_params,
timeout_seconds=timeout_to_seconds(timeout),
)
)
fields: Final = {
"model": model,
"audio": self._audio_payload(audio_file),
"api_key": api_key,
"api_base": api_base,
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
"optional_params": optional_params,
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, bound=fields)
return TranscriptionResponse(**rust(call))
return runtime.run(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
@ -85,18 +86,18 @@ class BedrockAudioTranscriptionRustDispatch:
timeout: float | httpx.Timeout | None,
) -> TranscriptionResponse:
async def native(rust: RustAtranscription) -> TranscriptionResponse:
return TranscriptionResponse(
**await rust(
model=model,
audio=self._audio_payload(audio_file),
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
optional_params=optional_params,
timeout_seconds=timeout_to_seconds(timeout),
)
)
fields: Final = {
"model": model,
"audio": self._audio_payload(audio_file),
"api_key": api_key,
"api_base": api_base,
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
"optional_params": optional_params,
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, bound=fields)
return TranscriptionResponse(**await rust(call))
return await runtime.arun(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),

36
litellm/messages/dispatch.py
View file

@ -1,22 +1,21 @@
import inspect
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.pass_through.messages import handler as main
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.messages.entrypoints import (
NATIVE_AMESSAGES,
NATIVE_MESSAGES,
LiteLLMMessagesRequest,
)
from litellm.rust_bridge.public_call import (
NativeCall,
bind,
optional_bool,
optional_mapping,
native_call,
native_call_hook,
optional_sequence,
optional_str,
signature,
@ -52,7 +51,7 @@ _AMESSAGES: Final = signature(_PYTHON_AMESSAGES)
def _public_request(
legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> LiteLLMMessagesRequest | None:
) -> NativeCall | None:
fields: Final = bind(legacy, args, kwargs)
if fields is None:
return None
@ -61,30 +60,21 @@ def _public_request(
max_tokens: Final = fields.get("max_tokens")
if not isinstance(model, str) or messages is None or not isinstance(max_tokens, int):
return None
return LiteLLMMessagesRequest(
model=model,
messages=messages,
max_tokens=max_tokens,
stream=optional_bool(fields.get("stream")),
api_key=optional_str(fields.get("api_key")),
api_base=optional_str(fields.get("api_base")),
custom_llm_provider=optional_str(fields.get("custom_llm_provider")),
kwargs=optional_mapping(fields.get("kwargs")) or MappingProxyType({}),
)
return native_call(args, kwargs, fields)
def _resolved_provider(request: LiteLLMMessagesRequest) -> str | None:
def _resolved_provider(request: NativeCall) -> str | None:
try:
return get_llm_provider(request.model, request.custom_llm_provider)[1]
return get_llm_provider(str(request.bound["model"]), optional_str(request.bound.get("custom_llm_provider")))[1]
except BadRequestError:
return request.custom_llm_provider
return optional_str(request.bound.get("custom_llm_provider"))
def _context(request: LiteLLMMessagesRequest) -> RouteContext:
def _context(request: NativeCall) -> RouteContext:
return RouteContext(
Route.MESSAGES,
provider=_resolved_provider(request),
model=request.model,
model=str(request.bound["model"]),
)
@ -112,7 +102,7 @@ def anthropic_messages_handler(
kwargs,
python=python,
binding=NATIVE_MESSAGES,
native=call_hook,
native=native_call_hook,
)
@ -123,7 +113,7 @@ async def anthropic_messages(*args: object, **kwargs: object) -> MessagesResult:
kwargs,
python=python,
binding=NATIVE_AMESSAGES,
native=call_hook,
native=native_call_hook,
)

183
litellm/model_prices_and_context_window_backup.json
View file

@ -42380,6 +42380,40 @@
"supports_response_schema": true,
"supports_web_search": true
},
"openrouter/anthropic/claude-haiku-5.5": {
"input_cost_per_token": 1e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_web_search": true,
"supports_adaptive_thinking": true,
"prompt_cache_min_tokens": 512,
"supports_sampling_params": false
},
"openrouter/anthropic/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_creation_input_token_cost_above_1hr": 2e-06,
@ -80770,6 +80804,10 @@
"supports_vision": true
},
"claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens_batches": 2.5e-07,
"output_cost_per_token_above_100k_tokens_batches": 1.25e-06,
"cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07,
"cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08,
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
@ -80809,8 +80847,8 @@
"us": 1.1
},
"supports_output_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview",
"supports_web_search": true,
@ -80821,6 +80859,11 @@
"cache_read_input_token_cost_above_100k_tokens": 5e-08
},
"bedrock_mantle/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
@ -80856,12 +80899,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
@ -80875,8 +80923,8 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
@ -80898,6 +80946,11 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
@ -80933,12 +80986,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"apac.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
@ -80964,8 +81022,8 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_creation_input_token_cost": 1.375e-07,
@ -80975,6 +81033,11 @@
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"au.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
@ -81010,12 +81073,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"azure_ai/claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
@ -81052,6 +81120,11 @@
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
@ -81065,9 +81138,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -81087,6 +81161,11 @@
"supports_xhigh_reasoning_effort": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
@ -81100,9 +81179,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -81122,6 +81202,11 @@
"supports_xhigh_reasoning_effort": true
},
"eu.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
@ -81157,12 +81242,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"global.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
@ -81198,12 +81288,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"jp.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
@ -81239,10 +81334,10 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"perplexity/anthropic/claude-haiku-5-5": {
"litellm_provider": "perplexity",
@ -81256,6 +81351,11 @@
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"us-gov.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
@ -81269,10 +81369,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -81292,6 +81392,11 @@
"supports_xhigh_reasoning_effort": true
},
"us.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
@ -81327,12 +81432,17 @@
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": false,
"thinking_always_on": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"vertex_ai/claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
@ -81370,9 +81480,14 @@
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"vertex_ai/claude-haiku-5-5@default": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
@ -81410,6 +81525,6 @@
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
}
}

1
litellm/models/mcp_server.py
View file

@ -111,6 +111,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
source_url: str | None = None
timeout: float | None = None
max_concurrent_requests: int | None = None
rpm: int | None = None
approval_status: str | None = Field(
default="active",
description="Approval status: 'pending_review', 'active', 'rejected'",

45
litellm/ocr/dispatch.py
View file

@ -1,4 +1,5 @@
from collections.abc import Coroutine, Mapping
from types import MappingProxyType
from typing import Final
import httpx
@ -6,8 +7,9 @@ import httpx
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR
from litellm.rust_bridge.public_call import NativeCall, native_call, native_call_hook, optional_str
__all__ = ("aocr", "ocr")
@ -21,30 +23,33 @@ def _bind_request(
custom_llm_provider: str | None = None,
extra_headers: dict[str, object] | None = None,
**kwargs: object, # kwargs-ok: public OCR accepts provider-specific options
) -> LiteLLMOcrRequest:
return LiteLLMOcrRequest(
model=model,
document=document,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
) -> Mapping[str, object]:
return MappingProxyType(
{
"model": model,
"document": document,
"api_key": api_key,
"api_base": api_base,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
"kwargs": kwargs,
}
)
def _public_request(name: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> LiteLLMOcrRequest:
def _public_request(name: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> NativeCall:
try:
return _bind_request(*args, **kwargs) # pyright: ignore[reportArgumentType] # Python binds the public arguments before native validation
fields: Final = _bind_request(*args, **kwargs) # pyright: ignore[reportArgumentType] # Python binds the public arguments before native validation
return native_call(args, kwargs, fields)
except TypeError as error:
raise TypeError(str(error).replace("_bind_request()", f"{name}()")) from None
def _context(request: LiteLLMOcrRequest) -> RouteContext:
prefix, separator, _ = request.model.partition("/")
provider: Final = request.custom_llm_provider or (prefix if separator else None)
return RouteContext(Route.OCR, provider=provider, model=request.model)
def _context(request: NativeCall) -> RouteContext:
prefix, separator, _ = str(request.bound["model"]).partition("/")
provider: Final = optional_str(request.bound.get("custom_llm_provider")) or (prefix if separator else None)
return RouteContext(Route.OCR, provider=provider, model=str(request.bound["model"]))
_DISPATCH: Final = PublicDispatch(
@ -70,7 +75,7 @@ def ocr(
kwargs,
python=runtime.NO_PYTHON,
binding=NATIVE_OCR,
native=call_hook,
native=native_call_hook,
)
@ -80,5 +85,5 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr
kwargs,
python=runtime.NO_PYTHON,
binding=NATIVE_AOCR,
native=call_hook,
native=native_call_hook,
)

19
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -1761,6 +1761,16 @@ def client_supplied_redirect_uris(value: object) -> list[str] | None:
return uris if len(uris) == len(value) else None
_CLIENT_APPLICATION_TYPE: Final = TypeAdapter(Literal["native", "web"] | None)
def client_supplied_application_type(value: object) -> Literal["native", "web"] | None:
try:
return _CLIENT_APPLICATION_TYPE.validate_python(value)
except ValidationError as exc:
raise HTTPException(status_code=400, detail="application_type must be native or web") from exc
async def _post_dcr_registration(
registration_url: str,
register_data: Mapping[str, object],
@ -1925,6 +1935,7 @@ async def register_client_with_server(
fallback_client_id: str | None = None,
persist_credentials: bool = False,
client_redirect_uris: list[str] | None = None,
client_application_type: Literal["native", "web"] | None = None,
):
_raise_if_not_oauth2(mcp_server)
request_base_url: Final = get_request_base_url(request)
@ -1980,6 +1991,11 @@ async def register_client_with_server(
)
register_data: Final = {
**(
{"application_type": client_application_type}
if bridge_relay and client_application_type is not None
else {}
),
"client_name": client_name,
"redirect_uris": client_redirect_uris if bridge_relay else [current_redirect_uri],
"grant_types": grant_types or (["authorization_code", "refresh_token"] if bridge_relay else []),
@ -3094,6 +3110,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
return await register_aggregate_client(
request=request, request_body=data, token_exchange_available=token_exchange_available()
)
client_application_type: Final = client_supplied_application_type(data.get("application_type"))
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
async with global_mcp_server_manager.catalog.operation():
@ -3115,6 +3132,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""),
fallback_client_id=resolved.server_name or resolved.name,
client_redirect_uris=client_redirect_uris,
client_application_type=client_application_type,
)
return dummy_return
@ -3130,4 +3148,5 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""),
fallback_client_id=mcp_server_name,
client_redirect_uris=client_redirect_uris,
client_application_type=client_application_type,
)

9
litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py
View file

@ -24,11 +24,13 @@ from litellm.proxy._experimental.mcp_server.exceptions import (
MCPUpstreamAuthError,
)
from litellm.proxy._experimental.mcp_server.faults.traversal import iter_exception_tree
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.types.llms.base import LiteLLMBaseModel
ListFaultCategory: TypeAlias = Literal[
"auth_required",
"forbidden",
"rate_limited",
"timeout",
"unreachable",
"upstream_error",
@ -64,6 +66,7 @@ class AggregateToolListing(NamedTuple):
tools: list[MCPTool]
outcomes: dict[str, ServerOutcome]
next_cursor: str | None = None
ttl_ms: int = 0
def _iter_upstream_responses(exc: BaseException) -> Iterator[httpx.Response | httpx2.Response]:
@ -125,6 +128,8 @@ def classify_list_exception(exc: BaseException) -> ServerListFault:
if isinstance(exc, MCPUpstreamAuthError):
tag: Final = "forbidden" if exc.status_code == 403 else "auth_required"
return ServerListFault(tag=tag, status_code=exc.status_code)
if isinstance(exc, ProxyRateLimitError):
return ServerListFault(tag="rate_limited", status_code=429)
if isinstance(exc, TimeoutError):
return ServerListFault(tag="timeout")
if isinstance(exc, ConnectionError):
@ -152,7 +157,7 @@ def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
match outcome.tag:
case "ok":
return {"status": "ok", "tool_count": outcome.tool_count}
case "auth_required" | "forbidden" | "timeout" | "unreachable" | "upstream_error" | "internal":
case "auth_required" | "forbidden" | "rate_limited" | "timeout" | "unreachable" | "upstream_error" | "internal":
return {
"status": outcome.tag,
**({"http_status": outcome.status_code} if outcome.status_code is not None else {}),
@ -170,6 +175,8 @@ def list_fault_http_status(fault: ServerListFault) -> int:
return fault.status_code or 401
case "forbidden":
return 403
case "rate_limited":
return 429
case "timeout":
return 504
case "unreachable" | "upstream_error":

191
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -29,7 +29,7 @@ from dataclasses import dataclass, replace
from functools import lru_cache
from itertools import chain, groupby
from types import EllipsisType, MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypedDict, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast
from urllib.parse import ParseResult, urlparse
import anyio
@ -45,15 +45,18 @@ from mcp.types import (
GetPromptResult,
InputRequiredResult,
ListPromptsRequest,
ListPromptsResult,
ListResourcesRequest,
ListResourcesResult,
ListResourceTemplatesRequest,
ListResourceTemplatesResult,
ListToolsResult,
PaginatedRequestParams,
Prompt,
ResourceTemplate,
)
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl, BaseModel, Field, TypeAdapter
from pydantic import AnyUrl, Field, TypeAdapter
from typing_extensions import ReadOnly, assert_never
import litellm
@ -78,6 +81,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPServerAccess,
_is_mcp_admitted_user_subject,
)
from litellm.proxy._experimental.mcp_server.catalog import _configuration_identity, _DiscoveryCache, _DiscoveryKey
from litellm.proxy._experimental.mcp_server.contracts import OperationContext
from litellm.proxy._experimental.mcp_server.elicitation_handler import (
MCP_ELICITATION_AVAILABLE,
@ -426,6 +430,7 @@ class MCPServerConfig(TypedDict, total=False):
client_assertion_signing_alg: str
timeout: float
max_concurrent_requests: int
rpm: ReadOnly[int | None]
class _ProtectedResourceMetadataPayload(TypedDict, total=False):
@ -1743,90 +1748,6 @@ def _record_mcp_guardrail_evaluations(
verbose_logger.warning("Failed to record MCP guardrail evaluation for logging: %s", e)
_DiscoveryItem = TypeVar("_DiscoveryItem", bound=BaseModel)
_DiscoveryKey: TypeAlias = tuple[str, str | None]
_DISCOVERY_CACHE_LIMIT: Final = 1024
class _DiscoveryCache(Generic[_DiscoveryItem]):
def __init__(
self, ttl: float, clock: Callable[[], float], adapter: TypeAdapter[tuple[_DiscoveryItem, ...]]
) -> None:
self._ttl = ttl
self._adapter = adapter
self._entries = InMemoryCache(max_size_in_memory=_DISCOVERY_CACHE_LIMIT, max_size_per_item=64, clock=clock)
self._pending: dict[_DiscoveryKey, asyncio.Task[list[_DiscoveryItem]]] = {}
self._waiters: dict[asyncio.Task[list[_DiscoveryItem]], int] = {} # mutable-ok: constant-time waiter accounting
def invalidate(self, server_id: str) -> None:
prefix: Final = f"[{json.dumps(server_id)},"
keys: Final = cast( # cast-ok: private cache contains only JSON string keys
"tuple[str, ...]", tuple(self._entries.cache_dict)
)
for entry_key in keys:
if entry_key.startswith(prefix):
self._entries.delete_cache(entry_key)
for key in tuple(self._pending):
if key[0] == server_id:
self._pending.pop(key)
@staticmethod
def _observe_completion(task: asyncio.Task[list[_DiscoveryItem]]) -> None:
if not task.cancelled():
task.exception()
async def get(
self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[list[_DiscoveryItem]]]
) -> tuple[_DiscoveryItem, ...]:
if self._ttl <= 0:
return tuple(await fetch())
entry: Final[object] = self._entries.get_cache(json.dumps(key))
if entry is not None:
return self._adapter.validate_python(entry)
pending: Final = self._pending.get(key)
if pending is not None:
return await self._await_fetch(key, pending)
if len(self._pending) >= _DISCOVERY_CACHE_LIMIT:
return tuple(await fetch())
task: Final = asyncio.create_task(self._fetch(key, fetch))
self._pending[key] = task
task.add_done_callback(self._observe_completion)
return await self._await_fetch(key, task)
async def _await_fetch(
self, key: _DiscoveryKey, task: asyncio.Task[list[_DiscoveryItem]]
) -> tuple[_DiscoveryItem, ...]:
self._waiters[task] = self._waiters.get(task, 0) + 1
try:
return tuple(item.model_copy(deep=True) for item in await asyncio.shield(task))
finally:
remaining: Final = self._waiters[task] - 1
if remaining:
self._waiters[task] = remaining
else:
self._waiters.pop(task)
if self._pending.get(key) is task:
self._pending.pop(key)
if not task.done():
task.cancel()
async def _fetch(
self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[list[_DiscoveryItem]]]
) -> list[_DiscoveryItem]:
try:
items: Final = await fetch()
if self._pending.get(key) is asyncio.current_task():
self._entries.set_cache(
json.dumps(key),
self._adapter.dump_json(tuple(items)),
ttl=self._ttl,
)
return items
finally:
if self._pending.get(key) is asyncio.current_task():
self._pending.pop(key)
def _mcp_discovery_cache_ttl() -> float:
raw: Final = os.environ.get("LITELLM_MCP_DISCOVERY_CACHE_TTL", "60")
try:
@ -1967,14 +1888,14 @@ class MCPServerManager:
token_exchanger=build_token_exchanger(),
)
discovery_ttl: Final = _mcp_discovery_cache_ttl()
self._prompt_discovery_cache = _DiscoveryCache[Prompt](
discovery_ttl, discovery_clock, TypeAdapter(tuple[Prompt, ...])
self._prompt_discovery_cache = _DiscoveryCache[ListPromptsResult](
discovery_ttl, discovery_clock, TypeAdapter(ListPromptsResult)
)
self._resource_discovery_cache = _DiscoveryCache[Resource](
discovery_ttl, discovery_clock, TypeAdapter(tuple[Resource, ...])
self._resource_discovery_cache = _DiscoveryCache[ListResourcesResult](
discovery_ttl, discovery_clock, TypeAdapter(ListResourcesResult)
)
self._template_discovery_cache = _DiscoveryCache[ResourceTemplate](
discovery_ttl, discovery_clock, TypeAdapter(tuple[ResourceTemplate, ...])
self._template_discovery_cache = _DiscoveryCache[ListResourceTemplatesResult](
discovery_ttl, discovery_clock, TypeAdapter(ListResourceTemplatesResult)
)
from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots
@ -2738,6 +2659,7 @@ class MCPServerManager:
allow_elicitation=bool(server_config.get("allow_elicitation", False)),
timeout=server_config.get("timeout", None),
max_concurrent_requests=server_config.get("max_concurrent_requests", None),
rpm=server_config.get("rpm", None),
token_validation=server_config.get("token_validation", None),
oauth_identity_binding=server_config.get("oauth_identity_binding", None),
)
@ -3327,6 +3249,7 @@ class MCPServerManager:
or "rfc8693",
timeout=getattr(mcp_server, "timeout", None),
max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None),
rpm=getattr(mcp_server, "rpm", None),
)
_warn_legacy_delegate_auth_if_applicable(new_server, source="database")
if register_oauth_discovery:
@ -4605,24 +4528,34 @@ class MCPServerManager:
stdio_env: dict[str, str] | None,
subject_token: str | None,
credential_fingerprint: str | None = None,
per_caller: bool = False,
raw_headers: Mapping[str, str] | None = None,
) -> _DiscoveryKey:
per_user: Final = (
per_caller
or server.requires_per_user_auth
or self._references_per_user_env_var(server)
or server.delegate_auth_to_upstream
or server.auth_type in (MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag)
)
if not (per_user or mcp_auth_header or extra_headers or stdio_env or subject_token):
return server.server_id, None
identity: Final = (
(user_api_key_auth.user_id, user_api_key_auth.api_key)
if per_user and user_api_key_auth is not None
user_api_key_auth.model_dump(
include={
"end_user_id",
"user_role",
"object_permission_id",
"team_object_permission_id",
"team_object_permission",
"end_user_object_permission",
},
mode="json",
)
if user_api_key_auth is not None
else None
)
material: Final = json.dumps(
(identity, mcp_auth_header, extra_headers, stdio_env, subject_token, credential_fingerprint),
(
_configuration_identity(server),
_admission_identity(user_api_key_auth, raw_headers) if user_api_key_auth is not None else None,
identity,
mcp_auth_header,
extra_headers,
stdio_env,
subject_token,
credential_fingerprint,
),
sort_keys=True,
separators=(",", ":"),
)
@ -4738,14 +4671,21 @@ class MCPServerManager:
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint
server,
user_api_key_auth,
mcp_auth_header,
headers,
stdio_env,
subject_token,
credential_fingerprint,
raw_headers=raw_headers,
)
async def fetch() -> list[Prompt]:
return await client.list_prompts(raise_on_error=True)
async def fetch() -> ListPromptsResult:
return await client.list_prompts_result(raise_on_error=True)
items: Final = await self._prompt_discovery_cache.get(key, fetch)
return self._create_prefixed_prompts(items, server, add_prefix=add_prefix)
return self._create_prefixed_prompts(items.prompts, server, add_prefix=add_prefix)
except Exception as error:
verbose_logger.warning("Failed to get prompts from server %s: %s", server.name, error)
return []
@ -4786,14 +4726,21 @@ class MCPServerManager:
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint
server,
user_api_key_auth,
mcp_auth_header,
headers,
stdio_env,
subject_token,
credential_fingerprint,
raw_headers=raw_headers,
)
async def fetch() -> list[Resource]:
return await client.list_resources(raise_on_error=True)
async def fetch() -> ListResourcesResult:
return await client.list_resources_result(raise_on_error=True)
items: Final = await self._resource_discovery_cache.get(key, fetch)
return self._create_prefixed_resources(items, server, add_prefix=add_prefix)
return self._create_prefixed_resources(items.resources, server, add_prefix=add_prefix)
except Exception as error:
verbose_logger.warning("Failed to get resources from server %s: %s", server.name, error)
return []
@ -4834,14 +4781,21 @@ class MCPServerManager:
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint
server,
user_api_key_auth,
mcp_auth_header,
headers,
stdio_env,
subject_token,
credential_fingerprint,
raw_headers=raw_headers,
)
async def fetch() -> list[ResourceTemplate]:
return await client.list_resource_templates(raise_on_error=True)
async def fetch() -> ListResourceTemplatesResult:
return await client.list_resource_templates_result(raise_on_error=True)
items: Final = await self._template_discovery_cache.get(key, fetch)
return self._create_prefixed_resource_templates(items, server, add_prefix=add_prefix)
return self._create_prefixed_resource_templates(items.resource_templates, server, add_prefix=add_prefix)
except Exception as error:
verbose_logger.warning("Failed to get resource_templates from server %s: %s", server.name, error)
return []
@ -5973,6 +5927,7 @@ class MCPServerManager:
data=synthetic_llm_data,
call_type=CallTypes.call_mcp_tool.value,
)
await proxy_logging_obj.enforce_mcp_server_rate_limits(user_api_key_auth, server)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = proxy_logging_obj._convert_mcp_hook_response_to_kwargs(modified_data, pre_hook_kwargs)
@ -7193,6 +7148,7 @@ class MCPServerManager:
instructions=server.instructions,
timeout=server.timeout,
max_concurrent_requests=server.max_concurrent_requests,
rpm=server.rpm,
)
async def get_all_mcp_servers_with_health_and_teams(
@ -7316,6 +7272,7 @@ class MCPServerManager:
instructions=server.instructions,
timeout=server.timeout,
max_concurrent_requests=server.max_concurrent_requests,
rpm=server.rpm,
)
async def get_all_mcp_servers_unfiltered(self) -> list[LiteLLM_MCPServerTable]:

100
litellm/proxy/_experimental/mcp_server/operations.py
View file

@ -5,6 +5,8 @@ import traceback
import types
import uuid
from collections.abc import Mapping, Sequence
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime
from functools import partial
from typing import Any, Final, NoReturn, TypeAlias, overload
@ -78,6 +80,7 @@ from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
AggregateToolListing,
ServerListOk,
ServerOutcome,
classify_list_exception,
outcome_wire_value,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
@ -128,6 +131,7 @@ from litellm.proxy._types import (
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
publish_auth_cache_invalidation,
)
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
get_chain_id_from_headers,
@ -224,6 +228,65 @@ class ListMCPToolsRestAPIResponseObject(MCPTool):
model_config = ConfigDict(arbitrary_types_allowed=True)
@dataclass(frozen=True, slots=True)
class _MCPServerRateLimitAdmission:
admitted_servers: tuple[MCPServer, ...]
rejected_servers: tuple[tuple[MCPServer, ProxyRateLimitError], ...]
_mcp_server_admission_memo: Final[ContextVar[dict[str, asyncio.Task[ProxyRateLimitError | None]] | None]] = ContextVar(
"mcp_server_admission_memo", default=None
)
async def _enforce_mcp_server_rate_limit(
user_api_key_auth: UserAPIKeyAuth | None,
server: MCPServer,
) -> None:
from litellm.proxy.proxy_server import proxy_logging_obj
if proxy_logging_obj is not None:
await proxy_logging_obj.enforce_mcp_server_rate_limits(user_api_key_auth, server)
async def _admit_mcp_servers(
servers: Sequence[MCPServer],
user_api_key_auth: UserAPIKeyAuth | None,
) -> _MCPServerRateLimitAdmission:
memo: Final = _mcp_server_admission_memo.get()
async def _server_rate_limit_error(server: MCPServer) -> ProxyRateLimitError | None:
try:
await _enforce_mcp_server_rate_limit(user_api_key_auth, server)
except ProxyRateLimitError as error:
return error
return None
async def _admit_server(server: MCPServer) -> tuple[MCPServer, ProxyRateLimitError | None]:
if memo is None:
return server, await _server_rate_limit_error(server)
admission_task: Final = memo.get(server.server_id)
if admission_task is not None:
return server, await admission_task
created_task: Final = asyncio.create_task(_server_rate_limit_error(server))
memo[server.server_id] = created_task
return server, await created_task
results: Final = await asyncio.gather(*(_admit_server(server) for server in servers))
return _MCPServerRateLimitAdmission(
admitted_servers=tuple(server for server, error in results if error is None),
rejected_servers=tuple((server, error) for server, error in results if error is not None),
)
async def _mcp_server_rate_limit_rejection(
server: MCPServer,
user_api_key_auth: UserAPIKeyAuth | None,
) -> ProxyRateLimitError | None:
admission: Final = await _admit_mcp_servers((server,), user_api_key_auth)
return admission.rejected_servers[0][1] if admission.rejected_servers else None
async def _build_virtual_call_logging_obj(
name: str,
arguments: dict[str, object],
@ -961,6 +1024,7 @@ async def _get_tools_from_mcp_servers(
protocol_version: str | None = None,
*,
record_listing: bool = False,
enforce_rate_limits: bool = True,
) -> AggregateToolListing:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@ -1093,20 +1157,37 @@ async def _get_tools_from_mcp_servers(
return page.tools, outcome
if params is None:
server_admission: Final = (
await _admit_mcp_servers(allowed_mcp_servers, user_api_key_auth)
if enforce_rate_limits
else _MCPServerRateLimitAdmission(tuple(allowed_mcp_servers), ())
)
if not server_admission.admitted_servers and server_admission.rejected_servers:
raise server_admission.rejected_servers[0][1]
admitted_servers: Final = server_admission.admitted_servers
results: Final = await asyncio.gather(
*(_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers)
*(_fetch_and_filter_server_tools(server) for server in admitted_servers)
)
aggregated = AggregateToolListing(
tools=[tool for tools, _ in results for tool in tools],
outcomes={
_aggregate_server_key(server): outcome for server, (_, outcome) in zip(allowed_mcp_servers, results)
_aggregate_server_key(server): outcome for server, (_, outcome) in zip(admitted_servers, results)
}
| {
_aggregate_server_key(server): classify_list_exception(error)
for server, error in server_admission.rejected_servers
},
)
else:
from litellm.proxy._experimental.mcp_server.catalog import aggregate_gateway_tools
aggregated = await aggregate_gateway_tools(
context, params, allowed_mcp_servers, _prefetched_oauth_creds, record_listing=record_listing
context,
params,
allowed_mcp_servers,
_prefetched_oauth_creds,
record_listing=record_listing,
enforce_rate_limits=enforce_rate_limits,
)
all_tools: Final = aggregated.tools
server_outcomes: Final = aggregated.outcomes
@ -1751,6 +1832,7 @@ async def _list_tools_before_first_call(
raw_headers=raw_headers,
client_ip=client_ip,
record_listing=False,
enforce_rate_limits=False,
)
except Exception as e: # noqa: BLE001 # best effort: resolution below answers as it did before
verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e)
@ -2464,6 +2546,7 @@ async def mcp_get_prompt(
user_api_key_auth=user_api_key_auth,
)
await _enforce_mcp_server_rate_limit(user_api_key_auth, server)
return await global_mcp_server_manager.get_prompt_from_server(
server=server,
user_api_key_auth=user_api_key_auth,
@ -2517,6 +2600,7 @@ async def mcp_read_resource(
user_api_key_auth=user_api_key_auth,
)
await _enforce_mcp_server_rate_limit(user_api_key_auth, server)
return await global_mcp_server_manager.read_resource_from_server(
server=server,
user_api_key_auth=user_api_key_auth,
@ -3125,6 +3209,7 @@ class GatewayOperations:
if context.mcp_proxy_mode
else (ListPromptsRequest(), ListResourcesRequest(), ListResourceTemplatesRequest())
)
memo_token: Final = _mcp_server_admission_memo.set({})
tasks: Final = (
asyncio.create_task(
_execute_handle_list_tools(
@ -3139,9 +3224,12 @@ class GatewayOperations:
try:
results: Final = await asyncio.gather(*tasks)
finally:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
try:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
finally:
_mcp_server_admission_memo.reset(memo_token)
return build_discovery(
configured=configured_versions(),
revision=context.protocol_version or "2025-11-25",

5
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -55,6 +55,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.responses.mcp.request_context import MCPRequestContext
if TYPE_CHECKING:
@ -737,6 +738,8 @@ if MCP_AVAILABLE:
"""
from litellm.proxy.proxy_server import proxy_logging_obj
if apply_tool_filters and proxy_logging_obj is not None:
await proxy_logging_obj.enforce_mcp_server_rate_limits(user_api_key_auth, server)
listed_generation: Final = global_mcp_server_manager.listed_tools_generation(server.server_id)
tools: Final = await _list_server_tools(
server,
@ -900,6 +903,8 @@ if MCP_AVAILABLE:
# matching status code and WWW-Authenticate challenge; that is what
# lets standards-compliant MCP clients run the upstream OAuth flow.
raise
except ProxyRateLimitError:
raise
except MCPServerListError as e:
fault: Final = classify_list_exception(e)
verbose_logger.info("Listing tools from %s failed with a %s fault", server.name, fault.tag)

Some files were not shown because too many files have changed in this diff Show more